diff --git a/Compression.Core/Dictionary/StackerSd4/StackerSd4BuildingBlock.cs b/Compression.Core/Dictionary/StackerSd4/StackerSd4BuildingBlock.cs new file mode 100644 index 000000000..7fb8ff544 --- /dev/null +++ b/Compression.Core/Dictionary/StackerSd4/StackerSd4BuildingBlock.cs @@ -0,0 +1,39 @@ +using System.Buffers.Binary; +using Compression.Core.FileSystems.Cvf; +using Compression.Registry; + +namespace Compression.Core.Dictionary.StackerSd4; + +/// +/// Stacker 4's SD-4 codec as a raw primitive: LZ77 over a 2720-byte window, Huffman-coded with a +/// run-length coded code table (docs/CVF-CODECS-ON-DISK.md). The building-block wrapper stores +/// the uncompressed size (32-bit little-endian) before one SD-4 stream covering the whole input. +/// +public sealed class StackerSd4BuildingBlock : IBuildingBlock { + /// + public string Id => "BB_StackerSd4"; + /// + public string DisplayName => "Stacker SD-4"; + /// + public string Description => "Stac Stacker 4 cluster codec: LZ77 (2720-byte window) with canonical Huffman codes and a run-length coded table"; + /// + public AlgorithmFamily Family => AlgorithmFamily.Dictionary; + + /// + public byte[] Compress(ReadOnlySpan data) { + var stream = Sd4Codec.Encode(data, level: 3); + var result = new byte[4 + stream.Length]; + BinaryPrimitives.WriteInt32LittleEndian(result, data.Length); + stream.CopyTo(result, 4); + return result; + } + + /// + public byte[] Decompress(ReadOnlySpan data) { + if (data.Length < 4) throw new InvalidDataException("Stacker SD-4: missing size header."); + var size = BinaryPrimitives.ReadInt32LittleEndian(data); + if (size < 0) throw new InvalidDataException("Stacker SD-4: negative size."); + var stream = data[4..].ToArray(); + return Sd4Codec.Decode(stream, stream.Length, size); + } +} diff --git a/Compression.Core/FileSystems/Cvf/ClusterMatchFinder.cs b/Compression.Core/FileSystems/Cvf/ClusterMatchFinder.cs new file mode 100644 index 000000000..54708d990 --- /dev/null +++ b/Compression.Core/FileSystems/Cvf/ClusterMatchFinder.cs @@ -0,0 +1,81 @@ +namespace Compression.Core.FileSystems.Cvf; + +/// +/// Hash-chain LZ77 match finder over one cluster, shared by the DS/JM and SD-4 encoders. Every match +/// it reports lies wholly inside the cluster: no CVF codec carries a dictionary from one cluster to +/// the next. +/// +internal sealed class ClusterMatchFinder { + private const int HashBits = 13; + private const int MinLength = 3; + + private readonly byte[] _data; + private readonly int[] _head = new int[1 << HashBits]; + private readonly int[] _previous; + private readonly int _maxDistance; + private readonly int _maxLength; + private readonly int _maxChain; + private readonly int _niceLength; + private int _inserted; + + /// The cluster. + /// The farthest distance the format can express. + /// The longest match the format can express. + /// Search effort, 1 (fastest) and up: how many earlier positions with the same + /// hash are tried, and how long a match must be to end the search early. + public ClusterMatchFinder(byte[] data, int maxDistance, int maxLength, int level) { + this._data = data; + this._previous = new int[data.Length]; + this._maxDistance = maxDistance; + this._maxLength = maxLength; + (this._maxChain, this._niceLength) = level switch { + <= 1 => (16, 64), + 2 => (64, 258), + 3 => (256, 1024), + _ => (1024, int.MaxValue), + }; + Array.Fill(this._head, -1); + } + + /// + /// The longest match for the bytes at , preferring the nearest on ties; + /// (0, 0) when there is none of at least three bytes. Positions before + /// are indexed first, so calls must come in ascending order. + /// + /// Where the match starts. + /// Exclusive bound the match may not run past; the cluster end when omitted. + public (int Length, int Distance) Find(int position, int end = int.MaxValue) { + this.IndexUpTo(position); + var data = this._data; + var limit = Math.Min(this._maxLength, Math.Min(data.Length, end) - position); + if (limit < MinLength) return (0, 0); + + int bestLength = 0, bestDistance = 0; + var candidate = this._head[Hash(data, position)]; + for (var chain = this._maxChain; candidate >= 0 && chain > 0; --chain, candidate = this._previous[candidate]) { + var distance = position - candidate; + if (distance > this._maxDistance) break; + if (data[candidate + bestLength] != data[position + bestLength]) continue; + + var length = 0; + while (length < limit && data[candidate + length] == data[position + length]) ++length; + if (length <= bestLength) continue; + + (bestLength, bestDistance) = (length, distance); + if (length == limit || length >= this._niceLength) break; + } + return bestLength >= MinLength ? (bestLength, bestDistance) : (0, 0); + } + + private void IndexUpTo(int position) { + var last = Math.Min(position, this._data.Length - MinLength + 1); + for (; this._inserted < last; ++this._inserted) { + var h = Hash(this._data, this._inserted); + this._previous[this._inserted] = this._head[h]; + this._head[h] = this._inserted; + } + } + + private static int Hash(byte[] data, int position) + => (int)(((uint)data[position] << 16 | (uint)data[position + 1] << 8 | data[position + 2]) * 2654435761u >> (32 - HashBits)); +} diff --git a/Compression.Core/FileSystems/Cvf/CvfLzCodec.cs b/Compression.Core/FileSystems/Cvf/CvfLzCodec.cs index 414453bd2..118ed3a0d 100644 --- a/Compression.Core/FileSystems/Cvf/CvfLzCodec.cs +++ b/Compression.Core/FileSystems/Cvf/CvfLzCodec.cs @@ -1,324 +1,135 @@ -#pragma warning disable CS1591 -using System; -using System.Collections.Generic; -using System.IO; +using System.Buffers.Binary; using Compression.Core.Deflate; -using Compression.Core.Streams; namespace Compression.Core.FileSystems.Cvf; /// -/// Per-cluster compression methods of the DoubleSpace/DriveSpace/Stacker CVF -/// family (see docs/CVF-CODECS-ON-DISK.md). and -/// are the Stacker 3 and Stacker 4 codecs. -/// -public enum CvfLzMethod { Stored, Ds, Jm, Auto, Sq, Sd4, Sd3 } - -/// -/// Genuine DoubleSpace/DriveSpace per-cluster compression codec (DS-0-x and -/// JM-0-x), verified byte-exact against the independent dmsdos decoder. The -/// bitstream packs bits LSB-first into little-endian 16-bit words; a cluster -/// payload is a 16-bit magic ("DS"=0x5344 / "MJ"=0x4D4A) + 16-bit -/// version + an LZ77 token stream terminated by the 0x113f sync. +/// Per-cluster codecs of the DoubleSpace / DriveSpace / DriveSpace 3 / Stacker compressed volumes, +/// implemented from docs/CVF-CODECS-ON-DISK.md. An MSDBL cluster payload starts with a 16-bit +/// magic and a 16-bit version — "DS" (DS-0-x), "JM" (JM-0-x) or "SQ" (SQ-0-0, +/// raw DEFLATE) — followed by the token stream; a Stacker payload carries no method field and is +/// recognised by its first word. /// public static class CvfLzCodec { - public const uint DS_0_0 = 0x00005344; - public const uint JM_0_0 = 0x00004D4A; - public const uint SQ_0_0 = 0x00005153; - - private const int DsMagic = 0x5344; - private const int JmMagic = 0x4D4A; - private const int SqMagic = 0x5153; - private const int Sync = 0x113f; - private const int MaxLen = 512; // safe for both (DS replen<=512, JM<=513) - - // ── bit writer: bit i -> word[i/16] bit i%16, words stored little-endian ─── - private sealed class BitWriter { - private readonly List _words = []; - private int _bit; - public void Put(int value, int n) { - for (var i = 0; i < n; i++, this._bit++) { - var w = this._bit >> 4; - while (this._words.Count <= w) this._words.Add(0); - if (((value >> i) & 1) != 0) this._words[w] |= (ushort)(1 << (this._bit & 15)); - } - } - public byte[] ToBytes() { - var b = new byte[this._words.Count * 2]; - for (var i = 0; i < this._words.Count; i++) { - b[i * 2] = (byte)this._words[i]; - b[i * 2 + 1] = (byte)(this._words[i] >> 8); - } - return b; - } - } - - // ── bit reader: faithful port of dmsdos dblb_* ───────────────────────────── - private sealed class BitReader { - private static readonly uint[] Mask = - [0x0, 0x1, 0x3, 0x7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, - 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF]; - private readonly byte[] _d; - private readonly int _len; - private uint _buf; - private int _pb = 32; - private int _pd; - private readonly int _pe; - public BitReader(byte[] d, int len) { this._d = d; this._len = len; this._pe = (len + 1) >> 1; } - private ushort Word(int i) { - var o = i * 2; - var lo = o < this._len ? this._d[o] : 0; - var hi = o + 1 < this._len ? this._d[o + 1] : 0; - return (ushort)(lo | (hi << 8)); - } - public uint Peek() { - if (this._pb >= 16) { - this._buf >>= 16; this._pb -= 16; - if (this._pd < this._pe) this._buf |= (uint)Word(this._pd++) << 16; - } - return this._buf >> this._pb; - } - public void Skip(int n) => this._pb += n; - public uint ReadN(int n) { var u = this.Peek(); this._pb += n; return u & Mask[n]; } + /// Method header of a DS-0-0 payload, read as a little-endian 32-bit word. + public const uint DS_0_0 = 'D' | 'S' << 8; - public int RdOffs() { - var u = this.Peek(); - switch (u & 3) { - case 0: case 2: this._pb += 1 + 6; return (int)(63 & (u >> 1)); - case 1: this._pb += 2 + 8; return (int)((255 & (u >> 2)) + 64); - default: this._pb += 2 + 12; return (int)((4095 & (u >> 2)) + 320); - } - } - public int RdLen() { - var u = this.Peek(); - switch (u & 15) { - case 1: case 3: case 5: case 7: case 9: case 11: case 13: case 15: this._pb += 1; return 3; - case 2: case 6: case 10: case 14: this._pb += 3; return (int)((1 & (u >> 2)) + 4); - case 4: case 12: this._pb += 5; return (int)((3 & (u >> 3)) + 6); - case 8: this._pb += 7; return (int)((7 & (u >> 4)) + 10); - } - switch ((u >> 4) & 15) { - case 1: case 3: case 5: case 7: case 9: case 11: case 13: case 15: this._pb += 9; return (int)((15 & (u >> 5)) + 18); - case 2: case 6: case 10: case 14: this._pb += 11; return (int)((31 & (u >> 6)) + 34); - case 4: case 12: this._pb += 13; return (int)((63 & (u >> 7)) + 66); - case 8: this._pb += 15; return (int)((127 & (u >> 8)) + 130); - } - this._pb += 9; - if ((u & 256) != 0) return (int)this.ReadN(8) + 258; - return -1; - } - } + /// Method header of a JM-0-0 payload, read as a little-endian 32-bit word. + public const uint JM_0_0 = 'J' | 'M' << 8; - // ── length / offset encoders (inverse of RdLen / RdOffs) ─────────────────── - private static void PutLen(BitWriter w, int v) { // v = rdlen value, 3..513 - var x = v - 2; - var c = 0; - while ((1 << (c + 1)) <= x) c++; - w.Put(1 << c, c + 1); // c zero bits then a 1 - w.Put(x - (1 << c), c); // c data bits - } - - private static void PutOffs(BitWriter w, int off) { - if (off <= 63) { w.Put(0, 1); w.Put(off, 6); } - else if (off <= 319) { w.Put(1, 2); w.Put(off - 64, 8); } - else { w.Put(3, 2); w.Put(off - 320, 12); } - } + /// Method header of an SQ-0-0 payload, read as a little-endian 32-bit word. + public const uint SQ_0_0 = 'S' | 'Q' << 8; - // =========================================================================== - // Compression - // =========================================================================== + private const ushort Sd4Signature = 0x0081; + private const int HeaderBytes = 4; /// - /// Compresses one cluster. Returns the payload (4-byte method header + token - /// stream, padded to a 2-byte word), or null if it would not be smaller - /// than the raw cluster (caller stores raw instead). + /// Compresses one cluster. Returns the payload, or when it would not be + /// smaller than the raw cluster (the caller then stores the cluster raw). /// public static byte[]? Compress(ReadOnlySpan data, CvfLzMethod method, int level) { - var bytes = Encode(data, method, level); - if (bytes is null) return null; - return bytes.Length < data.Length ? bytes : null; - } - - /// Encodes a cluster with the given method, always returning the - /// payload (4-byte header + token stream), or null for - /// / unsupported methods. The caller decides - /// whether the result fits the cluster's sector budget. - public static byte[]? Encode(ReadOnlySpan data, CvfLzMethod method, int level) { - switch (method) { - case CvfLzMethod.Ds: return CompressDs(data, level); - case CvfLzMethod.Jm: return CompressJm(data, level); - case CvfLzMethod.Sq: return CompressSq(data); - case CvfLzMethod.Sd4: return Sd4Codec.Encode(data); - case CvfLzMethod.Sd3: return Sd3Codec.Encode(data); - case CvfLzMethod.Auto: - // Per-cluster best-of: each cluster carries its own method header, so we - // pick whichever codec yields the smallest payload for this cluster. - byte[]?[] cands = [CompressDs(data, level), CompressJm(data, level), CompressSq(data)]; - byte[]? best = null; - foreach (var c in cands) if (c is not null && (best is null || c.Length < best.Length)) best = c; - return best; - default: return null; - } + var payload = Encode(data, method, level); + return payload is not null && payload.Length < data.Length ? payload : null; } - private static byte[] CompressDs(ReadOnlySpan data, int level) { - var w = new BitWriter(); - w.Put(DsMagic, 16); - w.Put(0, 16); // version 0 (DS-0-0) - var src = data.ToArray(); - var pos = 0; - while (pos < src.Length) { - var (mlen, moff) = FindMatch(src, pos, 4415, 2, level); - if (mlen >= 2) { - if (moff <= 63) w.Put(0 | (moff << 2), 8); - else if (moff <= 319) w.Put(3 | ((moff - 64) << 3), 11); - else w.Put(7 | ((moff - 320) << 3), 15); - PutLen(w, mlen + 1); // DS: copy length = rdlen - 1 - pos += mlen; - } else { - var b = src[pos++]; - if (b < 128) w.Put(2 | (b << 2), 9); - else w.Put(1 | ((b & 127) << 2), 9); - } - } - w.Put(7, 3); w.Put(Sync - 320, 12); // final sync - return w.ToBytes(); - } - - private static byte[] CompressJm(ReadOnlySpan data, int level) { - var w = new BitWriter(); - w.Put(JmMagic, 16); - w.Put(0, 16); // version 0 (JM-0-0) - var src = data.ToArray(); - var pos = 0; - while (pos < src.Length) { - var (mlen, moff) = FindMatch(src, pos, 4415, 3, level); - if (mlen >= 3) { - w.Put(1, 2); // match token - PutOffs(w, moff); - PutLen(w, mlen); // JM: copy length = rdlen - pos += mlen; - } else { - var b = src[pos++]; - if (b < 128) w.Put(b << 1, 8); - else w.Put(3 | ((b & 127) << 2), 9); - } - } - w.Put(1, 2); w.Put(3, 2); w.Put(Sync - 320, 12); // final sync (rdn(2)=1 + rdoffs 0x113f) - return w.ToBytes(); - } - - // SQ-0-0 (DriveSpace 3 "Ultra") is, byte-for-byte, "SQ" + 16-bit version + a - // raw RFC-1951 DEFLATE stream: dmsdos sq_dec's fixed-Huffman table, length and - // distance base/extra tables, and bit-reversed LSB-first packing all match - // DEFLATE exactly, so our Deflate encoder produces a genuine SQ body. - private static byte[] CompressSq(ReadOnlySpan data) { - using var ms = new MemoryStream(); - ms.WriteByte(0x53); ms.WriteByte(0x51); ms.WriteByte(0); ms.WriteByte(0); // "SQ", version 0 - using (var ds = new RawDeflateStream(ms, CompressionStreamMode.Compress, DeflateCompressionLevel.Default, leaveOpen: true)) - ds.Write(data); - return ms.ToArray(); - } - - private static byte[] DecompressSq(byte[] payload, int inLen, int outLen) { - using var ins = new MemoryStream(payload, 4, inLen - 4); - using var ds = new RawDeflateStream(ins, CompressionStreamMode.Decompress); - var outp = new byte[outLen]; - var n = 0; int r; - while (n < outLen && (r = ds.Read(outp, n, outLen - n)) > 0) n += r; - return outp; - } - - // Greedy longest-match finder. level scales the search depth (chain length). - private static (int Len, int Off) FindMatch(byte[] src, int pos, int maxOff, int minLen, int level) { - var bestLen = 0; var bestOff = 0; - var start = Math.Max(0, pos - maxOff); - var limit = Math.Min(MaxLen, src.Length - pos); - if (limit < minLen) return (0, 0); - var probes = level <= 0 ? 256 : level * 1024; - for (var cand = pos - 1; cand >= start && probes-- > 0; cand--) { - if (src[cand] != src[pos]) continue; - var l = 1; - while (l < limit && src[cand + l] == src[pos + l]) l++; - if (l > bestLen) { bestLen = l; bestOff = pos - cand; if (l >= limit) break; } - } - return bestLen >= minLen ? (bestLen, bestOff) : (0, 0); - } - - // =========================================================================== - // Decompression - // =========================================================================== + /// + /// Encodes a cluster with the given method and always returns the payload, or + /// for . The caller decides whether the + /// result fits the cluster's sector budget. keeps the smaller of DS + /// and JM. + /// + public static byte[]? Encode(ReadOnlySpan data, CvfLzMethod method, int level) => method switch { + CvfLzMethod.Ds => EncodeMsdbl(data, MsdblDialect.Ds, level), + CvfLzMethod.Jm => EncodeMsdbl(data, MsdblDialect.Jm, level), + CvfLzMethod.Auto => Smaller(EncodeMsdbl(data, MsdblDialect.Ds, level), EncodeMsdbl(data, MsdblDialect.Jm, level)), + CvfLzMethod.Sq => EncodeSq(data, level), + CvfLzMethod.Sd3 => Sd3Codec.Encode(data), + CvfLzMethod.Sd4 => Sd4Codec.Encode(data, level), + _ => null, + }; - /// Decompresses a cluster payload to exactly bytes. + /// + /// Decompresses an MSDBL cluster payload (DS, JM or SQ, told apart by its header) to exactly + /// bytes. + /// + /// Unknown method header or a malformed stream. public static byte[] Decompress(byte[] payload, int inLen, int outLen) { - var method = (uint)(payload[0] | (payload[1] << 8) | (payload[2] << 16) | (payload[3] << 24)); - var magic16 = (int)(method & 0xFFFF); - if (magic16 == SqMagic) return DecompressSq(payload, inLen, outLen); - if (magic16 == 0x0081) return Sd4Codec.Decode(payload, inLen, outLen); - var br = new BitReader(payload, inLen); - var read = br.ReadN(16); - if (read != (uint)magic16) throw new InvalidDataException($"CVF codec: bad magic 0x{read:X4}"); - br.ReadN(16); // version - var outp = new byte[outLen]; - var p = 0; - if (magic16 == DsMagic) DecodeDs(br, outp, ref p); - else if (magic16 == JmMagic) DecodeJm(br, outp, ref p); - else throw new InvalidDataException($"CVF codec: unsupported method 0x{method:X8}"); - return outp; + var input = payload.AsSpan(0, Math.Min(inLen, payload.Length)); + if (input.Length < HeaderBytes) throw new InvalidDataException("CVF: cluster payload shorter than its method header."); + + var output = new byte[outLen]; + var stream = input[HeaderBytes..]; + switch (BinaryPrimitives.ReadUInt16LittleEndian(input)) { + case (ushort)DS_0_0: MsdblLzCodec.Decode(stream, output, MsdblDialect.Ds, stopAtEndOfInput: false); break; + case (ushort)JM_0_0: MsdblLzCodec.Decode(stream, output, MsdblDialect.Jm, stopAtEndOfInput: false); break; + case (ushort)SQ_0_0: DecodeSq(stream, output); break; + case var magic: throw new InvalidDataException($"CVF: unknown cluster method header 0x{magic:X4}."); + } + return output; } /// /// Decompresses a Stacker cluster payload to bytes. Stacker records no /// method per cluster; the payload's first little-endian word selects it — 0x0081 SD-4, - /// 0x5344 ("DS") the DoubleSpace codec, anything else SD-3, whose first token is always a + /// 0x5344 ("DS") the DoubleSpace grammar, anything else SD-3, whose first token is always a /// literal and so can never begin with either word. /// + /// A malformed stream. public static byte[] DecompressStacker(byte[] payload, int inLen, int outLen) { - ArgumentNullException.ThrowIfNull(payload); - if (inLen < 2) throw new InvalidDataException("Stacker cluster payload is too short."); - return (payload[0] | (payload[1] << 8)) switch { - 0x0081 => Sd4Codec.Decode(payload, inLen, outLen), - DsMagic => Decompress(payload, inLen, outLen), - _ => Sd3Codec.Decode(payload.AsSpan(0, inLen), outLen), - }; + var input = payload.AsSpan(0, Math.Min(inLen, payload.Length)); + var word = input.Length >= 2 ? BinaryPrimitives.ReadUInt16LittleEndian(input) : -1; + switch (word) { + case Sd4Signature: + return Sd4Codec.Decode(payload, input.Length, outLen); + case (int)DS_0_0 when input.Length >= HeaderBytes: + var output = new byte[outLen]; + MsdblLzCodec.Decode(input[HeaderBytes..], output, MsdblDialect.Ds, stopAtEndOfInput: true); + return output; + default: + return Sd3Codec.Decode(input, outLen); + } } - private static void DecodeDs(BitReader br, byte[] outp, ref int p) { - while (p < outp.Length) { - var u = br.Peek(); - switch (u & 3) { - case 0: br.Skip(8); Copy(outp, ref p, (int)((u >> 2) & 63), br, -1); break; - case 1: br.Skip(9); outp[p++] = (byte)((u >> 2) | 128); break; - case 2: br.Skip(9); outp[p++] = (byte)((u >> 2) & 127); break; - default: - int off; - if ((u & 4) != 0) { br.Skip(15); off = (int)((u >> 3) & 4095) + 320; } - else { br.Skip(11); off = (int)((u >> 3) & 255) + 64; } - Copy(outp, ref p, off, br, -1); - break; - } - } + private static byte[] EncodeMsdbl(ReadOnlySpan data, MsdblDialect dialect, int level) { + var writer = new LsbBitWriter(); + WriteHeader(writer, dialect == MsdblDialect.Ds ? DS_0_0 : JM_0_0); + MsdblLzCodec.Encode(data, writer, dialect, level); + return writer.ToArray(alignment: 2); } - private static void DecodeJm(BitReader br, byte[] outp, ref int p) { - while (p < outp.Length) { - var u = br.Peek(); - switch (u & 3) { - case 0: case 2: br.Skip(8); outp[p++] = (byte)((u >> 1) & 127); break; - case 1: br.Skip(2); Copy(outp, ref p, br.RdOffs(), br, 0); break; - default: br.Skip(9); outp[p++] = (byte)(((u >> 2) & 127) | 128); break; + private static byte[] EncodeSq(ReadOnlySpan data, int level) { + var deflateLevel = level switch { + <= 1 => DeflateCompressionLevel.Fast, + 2 => DeflateCompressionLevel.Default, + 3 => DeflateCompressionLevel.Best, + _ => DeflateCompressionLevel.Maximum, + }; + var stream = DeflateCompressor.Compress(data, deflateLevel); + var payload = new byte[HeaderBytes + stream.Length + (stream.Length & 1)]; + BinaryPrimitives.WriteUInt32LittleEndian(payload, SQ_0_0); + stream.CopyTo(payload, HeaderBytes); + return payload; + } + + // The DEFLATE stream ends at its final block or when the cluster is full, whichever comes first. + private static void DecodeSq(ReadOnlySpan stream, Span output) { + try { + using var input = new MemoryStream(stream.ToArray(), writable: false); + var inflater = new DeflateDecompressor(input); + var filled = 0; + while (filled < output.Length) { + var n = inflater.Decompress(output[filled..]); + if (n == 0) break; + filled += n; } + } catch (Exception e) when (e is IOException or ArgumentException or IndexOutOfRangeException) { + throw new InvalidDataException("SQ: malformed DEFLATE stream.", e); } } - private static void Copy(byte[] outp, ref int p, int off, BitReader br, int k) { - if (off == Sync) return; // sync token mid-stream (shouldn't fill) - var len = br.RdLen() + k; - // Past the end of the input every token reads as a zero-distance, invalid-length match, so - // these checks also guarantee that a truncated stream terminates. - if (len <= 0) throw new InvalidDataException("CVF codec: invalid match length."); - if (off <= 0 || off > p) throw new InvalidDataException("CVF codec: match reaches before the start of the cluster."); - var src = p - off; - for (var i = 0; i < len && p < outp.Length; i++) outp[p++] = outp[src + i]; + private static void WriteHeader(LsbBitWriter writer, uint header) { + for (var i = 0; i < HeaderBytes; ++i) writer.WriteByte((byte)(header >> (8 * i))); } + + private static byte[] Smaller(byte[] a, byte[] b) => b.Length < a.Length ? b : a; } diff --git a/Compression.Core/FileSystems/Cvf/CvfLzMethod.cs b/Compression.Core/FileSystems/Cvf/CvfLzMethod.cs new file mode 100644 index 000000000..acfcc979d --- /dev/null +++ b/Compression.Core/FileSystems/Cvf/CvfLzMethod.cs @@ -0,0 +1,23 @@ +namespace Compression.Core.FileSystems.Cvf; + +/// +/// Per-cluster compression methods of the DoubleSpace/DriveSpace/Stacker CVF family (see +/// docs/CVF-CODECS-ON-DISK.md). and are the Stacker 3 and +/// Stacker 4 codecs. +/// +public enum CvfLzMethod { + /// No compression; the cluster is stored as is. + Stored = 0, + /// DoubleSpace DS-0-x LZ77. + Ds = 1, + /// DriveSpace JM-0-x LZ77 (DriveSpace 3 Normal/High). + Jm = 2, + /// Per cluster the smaller of and , both of which every MSDBL volume reads. + Auto = 3, + /// DriveSpace 3 SQ-0-0 (Ultra): a raw DEFLATE stream behind the method header. + Sq = 4, + /// Stacker 4 SD-4: Huffman-coded LZ77 behind the header word 0x0081. + Sd4 = 5, + /// Stacker 3 SD-3: a bare LZS stream closed by a check byte. + Sd3 = 6, +} diff --git a/Compression.Core/FileSystems/Cvf/MsdblLzCodec.cs b/Compression.Core/FileSystems/Cvf/MsdblLzCodec.cs new file mode 100644 index 000000000..ba8d954ca --- /dev/null +++ b/Compression.Core/FileSystems/Cvf/MsdblLzCodec.cs @@ -0,0 +1,268 @@ +namespace Compression.Core.FileSystems.Cvf; + +/// The two Microsoft LZ77 token grammars of docs/CVF-CODECS-ON-DISK.md. +internal enum MsdblDialect { + /// DoubleSpace DS-0-x: two-bit literal tags, matches copy one byte less than their length code. + Ds, + /// DriveSpace JM-0-x: one-bit tag for low literals, matches copy their full length code. + Jm, +} + +/// +/// The DS-0-x / JM-0-x token stream (without the 4-byte method header): bits LSB-first from +/// consecutive bytes, literals and back-references of up to 4414 bytes distance, and the sync marker +/// (distance 4415) that closes every cluster. +/// +internal static class MsdblLzCodec { + /// The all-ones 12-bit long distance, which is the sync marker rather than a match. + public const int SyncDistance = 320 + 0xFFF; + + /// The farthest real match. + public const int MaxDistance = SyncDistance - 1; + + /// The longest length code: nine-step prefix, 2 + 2^8 + 255. + private const int MaxLengthCode = 513; + + private const int MaxLengthPrefix = 8; + + /// Decoded bytes between two sync markers in an encoded cluster: one sector. + private const int SyncInterval = 512; + + // ── decoding ──────────────────────────────────────────────────────────── + + /// + /// Decodes the token stream in until is full. + /// + /// The token stream, starting right after the method header. + /// The cluster; bytes not written stay as they are (zero). + /// DS or JM. + /// + /// Stacker's use of the DS grammar: a stream that runs out (or reaches its zero padding) before the + /// cluster is full ends there instead of being an error. + /// + /// The number of bytes decoded. + /// The stream is malformed or ends early. + public static int Decode(ReadOnlySpan stream, Span output, MsdblDialect dialect, bool stopAtEndOfInput) { + var reader = new LsbBitReader(stream); + var position = 0; + while (position < output.Length) { + var token = dialect == MsdblDialect.Ds ? ReadDsToken(ref reader) : ReadJmToken(ref reader); + if (reader.Overran) { + if (stopAtEndOfInput) break; + throw new InvalidDataException($"{dialect}: token stream ends after {position} of {output.Length} bytes."); + } + + if (token.Distance == 0) { // literal (a real match never has distance 0) + if (token.Length != 0) { + if (stopAtEndOfInput) break; // zero padding read as a distance-0 match + throw new InvalidDataException($"{dialect}: match with distance 0."); + } + output[position++] = token.Literal; + continue; + } + + if (token.Distance == SyncDistance) { + // A cluster whose data ends before its nominal size is closed by a sync with nothing but + // padding behind it; the remainder of the cluster reads as zero. + if (reader.RestIsZero) break; + continue; + } + + var length = ReadLengthCode(ref reader); + if (reader.Overran) { + if (stopAtEndOfInput) break; + throw new InvalidDataException($"{dialect}: token stream ends inside a match."); + } + if (length < 0) throw new InvalidDataException($"{dialect}: length code has more than {MaxLengthPrefix} leading zeros."); + if (dialect == MsdblDialect.Ds) --length; + if (token.Distance > position) throw new InvalidDataException($"{dialect}: match reaches before the start of the cluster."); + + length = Math.Min(length, output.Length - position); + for (var end = position + length; position < end; ++position) + output[position] = output[position - token.Distance]; + } + return position; + } + + /// A literal ( 0, 0) or the distance of a match/sync. + private readonly record struct Token(byte Literal, int Distance, int Length = 0); + + private static Token ReadDsToken(ref LsbBitReader reader) { + var first = reader.Read(1); + var second = reader.Read(1); + return (first, second) switch { + (0, 0) => Match(reader.Read(6)), + (1, 0) => new((byte)(0x80 | reader.Read(7)), 0), + (0, 1) => new((byte)reader.Read(7), 0), + _ => reader.Read(1) == 0 ? Match(64 + reader.Read(8)) : Match(320 + reader.Read(12)), + }; + } + + private static Token ReadJmToken(ref LsbBitReader reader) { + if (reader.Read(1) == 0) return new((byte)reader.Read(7), 0); + if (reader.Read(1) == 1) return new((byte)(0x80 | reader.Read(7)), 0); + if (reader.Read(1) == 0) return Match(reader.Read(6)); + return reader.Read(1) == 0 ? Match(64 + reader.Read(8)) : Match(320 + reader.Read(12)); + } + + // Distance 0 cannot be a literal here: flag it with a non-zero length so the caller rejects it. + private static Token Match(int distance) => new(0, distance, distance == 0 ? 1 : 0); + + /// Reads z zeros, a one, then z bits: 2 + 2^z + v; -1 for nine zeros. + private static int ReadLengthCode(ref LsbBitReader reader) { + var prefix = 0; + while (reader.Read(1) == 0) { + if (++prefix > MaxLengthPrefix) return -1; + if (reader.Overran) return 0; + } + return 2 + (1 << prefix) + reader.Read(prefix); + } + + // ── encoding ──────────────────────────────────────────────────────────── + + /// + /// Encodes as a token stream with a sync marker after every 512 decoded + /// bytes and after the last one, appended to . As in the streams the dmsdos + /// compressors write, no match runs across a 512-byte boundary (distances still reach back across + /// them), so every sector of the cluster ends on a sync. + /// + public static void Encode(ReadOnlySpan data, LsbBitWriter writer, MsdblDialect dialect, int level) { + // A DS match copies one byte less than its length code, so its shortest copy is two bytes and its + // longest 512; the finder reports matches of three bytes and more. + var maxCopy = dialect == MsdblDialect.Ds ? MaxLengthCode - 1 : MaxLengthCode; + var copyBias = dialect == MsdblDialect.Ds ? 1 : 0; + var bytes = data.ToArray(); + var finder = new ClusterMatchFinder(bytes, MaxDistance, maxCopy, level); + + var sectorStart = 0; + do { + var sectorEnd = Math.Min(sectorStart + SyncInterval, bytes.Length); + var position = sectorStart; + var pending = finder.Find(position, sectorEnd); + while (position < sectorEnd) { + var (length, distance) = pending; + if (length == 0) { + WriteLiteral(writer, dialect, bytes[position]); + pending = ++position < sectorEnd ? finder.Find(position, sectorEnd) : default; + continue; + } + + // One step of lazy evaluation: a longer match starting at the next byte wins over this one. + var next = position + 1 < sectorEnd ? finder.Find(position + 1, sectorEnd) : default; + if (next.Length > length) { + WriteLiteral(writer, dialect, bytes[position]); + ++position; + pending = next; + continue; + } + + WriteMatch(writer, dialect, distance, length + copyBias); + position += length; + pending = position < sectorEnd ? finder.Find(position, sectorEnd) : default; + } + WriteSync(writer, dialect); + sectorStart = sectorEnd; + } while (sectorStart < bytes.Length); + } + + private static void WriteLiteral(LsbBitWriter writer, MsdblDialect dialect, byte value) { + var high = value >= 0x80; + if (dialect == MsdblDialect.Ds) + writer.Write(high ? 0b01 : 0b10, 2); // read order: "1 0" high, "0 1" low + else if (high) + writer.Write(0b11, 2); + else + writer.Write(0, 1); + writer.Write(value & 0x7F, 7); + } + + private static void WriteMatch(LsbBitWriter writer, MsdblDialect dialect, int distance, int lengthCode) { + WriteDistance(writer, dialect, distance); + var prefix = 31 - int.LeadingZeroCount(lengthCode - 2); + writer.Write(1 << prefix, prefix + 1); // `prefix` zeros, then a one + writer.Write(lengthCode - 2 - (1 << prefix), prefix); + } + + private static void WriteSync(LsbBitWriter writer, MsdblDialect dialect) => WriteDistance(writer, dialect, SyncDistance); + + private static void WriteDistance(LsbBitWriter writer, MsdblDialect dialect, int distance) { + if (dialect == MsdblDialect.Ds) { + switch (distance) { + case < 64: writer.Write(0b00, 2); writer.Write(distance, 6); break; + case < 320: writer.Write(0b011, 3); writer.Write(distance - 64, 8); break; + default: writer.Write(0b111, 3); writer.Write(distance - 320, 12); break; + } + return; + } + + writer.Write(0b01, 2); // read order: "1 0" + switch (distance) { + case < 64: writer.Write(0, 1); writer.Write(distance, 6); break; + case < 320: writer.Write(0b01, 2); writer.Write(distance - 64, 8); break; + default: writer.Write(0b11, 2); writer.Write(distance - 320, 12); break; + } + } +} + +/// Reads bits LSB-first from consecutive bytes; past the end it yields zeros and records the overrun. +internal ref struct LsbBitReader { + private readonly ReadOnlySpan _data; + private int _next; + private ulong _buffer; + private int _count; + + public LsbBitReader(ReadOnlySpan data) => this._data = data; + + /// True once a read has needed bits beyond the end of the data. + public readonly bool Overran => (long)this._next * 8 - this._count > (long)this._data.Length * 8; + + /// True when every bit not yet read is zero. + public readonly bool RestIsZero => this._buffer == 0 && (this._next >= this._data.Length || !this._data[this._next..].ContainsAnyExcept((byte)0)); + + /// Reads (at most 24) bits, the first read becoming the least significant. + public int Read(int count) { + while (this._count < count) { + if (this._next < this._data.Length) this._buffer |= (ulong)this._data[this._next] << this._count; + ++this._next; + this._count += 8; + } + var value = (int)(this._buffer & ((1UL << count) - 1)); + this._buffer >>= count; + this._count -= count; + return value; + } +} + +/// Writes bits LSB-first into consecutive bytes. +internal sealed class LsbBitWriter { + private byte[] _bytes = new byte[256]; + private int _length; + private ulong _accumulator; + private int _count; + + /// Appends a whole byte; only valid at a byte boundary. + public void WriteByte(byte value) => this.Write(value, 8); + + /// Appends the low (at most 32) bits of , least significant first. + public void Write(int value, int count) { + this._accumulator |= ((ulong)(uint)value & ((1UL << count) - 1)) << this._count; + this._count += count; + while (this._count >= 8) { + this.Push((byte)this._accumulator); + this._accumulator >>= 8; + this._count -= 8; + } + } + + /// Flushes the partial byte and pads with zero bytes to a multiple of . + public byte[] ToArray(int alignment) { + if (this._count > 0) this.Write(0, 8 - this._count); + while (this._length % alignment != 0) this.Push(0); + return this._bytes.AsSpan(0, this._length).ToArray(); + } + + private void Push(byte value) { + if (this._length == this._bytes.Length) Array.Resize(ref this._bytes, this._bytes.Length * 2); + this._bytes[this._length++] = value; + } +} diff --git a/Compression.Core/FileSystems/Cvf/PackageMerge.cs b/Compression.Core/FileSystems/Cvf/PackageMerge.cs new file mode 100644 index 000000000..1074b827c --- /dev/null +++ b/Compression.Core/FileSystems/Cvf/PackageMerge.cs @@ -0,0 +1,65 @@ +namespace Compression.Core.FileSystems.Cvf; + +/// +/// Optimal length-limited prefix code lengths by the package-merge algorithm (Larmore and Hirschberg, +/// "A fast algorithm for optimal length-limited Huffman codes", JACM 37(3), 1990). The result is +/// always a complete code — Kraft sum exactly one — which SD-4 requires of both its codes. +/// +internal static class PackageMerge { + /// + /// Code lengths for (zero frequency = unused, length 0), none longer + /// than . + /// + /// Fewer than two used symbols, or more than 2^maxLength. + public static int[] Lengths(ReadOnlySpan frequencies, int maxLength) { + var lengths = new int[frequencies.Length]; + var leaves = new List(); + for (var s = 0; s < frequencies.Length; ++s) + if (frequencies[s] > 0) leaves.Add(s); + if (leaves.Count < 2 || leaves.Count > 1L << maxLength) + throw new ArgumentException($"package-merge needs 2..2^{maxLength} used symbols, got {leaves.Count}.", nameof(frequencies)); + + var freq = frequencies.ToArray(); + leaves.Sort((a, b) => freq[a] != freq[b] ? freq[a].CompareTo(freq[b]) : a.CompareTo(b)); + + // Nodes: a leaf (Symbol >= 0) or a package of two nodes from the previous level. + var nodes = new List<(long Weight, int Symbol, int Left, int Right)>(); + var leafNodes = new List(leaves.Count); + foreach (var s in leaves) { + leafNodes.Add(nodes.Count); + nodes.Add((freq[s], s, -1, -1)); + } + + var level = leafNodes; + for (var depth = 1; depth < maxLength; ++depth) { + var packages = new List(level.Count / 2); + for (var i = 0; i + 1 < level.Count; i += 2) { + packages.Add(nodes.Count); + nodes.Add((nodes[level[i]].Weight + nodes[level[i + 1]].Weight, -1, level[i], level[i + 1])); + } + + // Merge the packages into the leaves by weight; leaves first on ties. + var merged = new List(leafNodes.Count + packages.Count); + int a = 0, b = 0; + while (a < leafNodes.Count || b < packages.Count) + merged.Add(b >= packages.Count || (a < leafNodes.Count && nodes[leafNodes[a]].Weight <= nodes[packages[b]].Weight) + ? leafNodes[a++] + : packages[b++]); + level = merged; + } + + // Each appearance of a leaf among the first 2n - 2 items adds one bit to its code. + var stack = new Stack(); + for (var i = 0; i < 2 * leaves.Count - 2; ++i) stack.Push(level[i]); + while (stack.Count > 0) { + var node = nodes[stack.Pop()]; + if (node.Symbol >= 0) { + ++lengths[node.Symbol]; + continue; + } + stack.Push(node.Left); + stack.Push(node.Right); + } + return lengths; + } +} diff --git a/Compression.Core/FileSystems/Cvf/Sd4Codec.cs b/Compression.Core/FileSystems/Cvf/Sd4Codec.cs index 394d2c0d9..58c188081 100644 --- a/Compression.Core/FileSystems/Cvf/Sd4Codec.cs +++ b/Compression.Core/FileSystems/Cvf/Sd4Codec.cs @@ -1,315 +1,418 @@ -#pragma warning disable CS1591 -using System; -using System.IO; - namespace Compression.Core.FileSystems.Cvf; /// -/// Genuine Stacker 4 (SD-4, cluster header 0x0081) per-cluster codec, as -/// specified in docs/CVF-CODECS-ON-DISK.md. SD-4 is Huffman-coded LZ77: a -/// helper code (22 symbols: lengths 0..15 and six run tokens) transmits the 0x150 -/// code lengths of the main code, whose symbols are the 256 literals, 63 short-match -/// tokens, 16 long-match tokens and the end token. -/// -/// reads the complete format — run tokens, short and long -/// matches with length extensions — and is pinned to streams the dmsdos SD-4 -/// encoder wrote. emits an all-literals stream (main code -/// over the cluster's byte frequencies, no match tokens), which the dmsdos driver -/// reads back byte-exact. The bitstream is MSB-first within little-endian 16-bit -/// words; codes are canonical and MSB-first. -/// +/// Stacker 4 (SD-4, cluster header 0x0081) per-cluster codec, implemented from +/// docs/CVF-CODECS-ON-DISK.md. SD-4 is Huffman-coded LZ77: a helper code (code lengths 0..15 +/// and six run tokens) transmits the 336 code lengths of the main code, whose symbols are the 256 +/// literals, 63 short-match tokens, 16 long-match tokens and the end token. Bits are read MSB-first +/// from little-endian 16-bit words; both codes are canonical and complete. /// public static class Sd4Codec { - private const int MaxLen = 10; // keep table2 codes within the decoder's fast table (bn<=10) - - // ── MSB-first bit writer into 16-bit little-endian words ─────────────────── - private sealed class BitWriter { - private readonly System.Collections.Generic.List _words = []; - private uint _cur; - private int _bits; - public void Put(int value, int n) { - for (var i = n - 1; i >= 0; i--) { - this._cur = (this._cur << 1) | (uint)((value >> i) & 1); - if (++this._bits == 16) { this._words.Add((ushort)this._cur); this._cur = 0; this._bits = 0; } - } - } - public byte[] ToBytes() { - if (this._bits > 0) { this._cur <<= 16 - this._bits; this._words.Add((ushort)this._cur); } - var b = new byte[this._words.Count * 2]; - for (var i = 0; i < this._words.Count; i++) { b[i * 2] = (byte)this._words[i]; b[i * 2 + 1] = (byte)(this._words[i] >> 8); } - return b; - } - } - - // ── MSB-first bit reader (mirror of sd4b_rd*) ────────────────────────────── - private sealed class BitReader { - private readonly byte[] _d; private readonly int _len; private long _buf; private int _pb; private int _pd; private readonly int _pe; - public BitReader(byte[] d, int len) { this._d = d; this._len = len; this._pe = (len + 1) >> 1; } - private ushort Word(int i) { var o = i * 2; var lo = o < this._len ? this._d[o] : 0; var hi = o + 1 < this._len ? this._d[o + 1] : 0; return (ushort)(lo | (hi << 8)); } - public int ReadN(int n) { - if (this._pb < n) { this._buf = (this._buf << 16) | (this._pd < this._pe ? Word(this._pd++) : 0u); this._pb += 16; if (this._pb < n) { this._buf = (this._buf << 16) | (this._pd < this._pe ? Word(this._pd++) : 0u); this._pb += 16; } } - this._pb -= n; - return (int)((this._buf >> this._pb) & ((1u << n) - 1)); - } - } - - // ── canonical Huffman: lengths -> (code,len) per symbol, matching sd4b_rdhufi - private static (int[] Code, int[] Len) Canonical(byte[] lengths) { - var cdLn = new int[17]; - foreach (var l in lengths) if (l is > 0 and < 16) cdLn[l]++; - var firstCode = new int[17]; - int us = 0; - for (var i = 1; i <= 16; i++) { firstCode[i] = us; us += cdLn[i]; us <<= 1; } - var next = (int[])firstCode.Clone(); - var code = new int[lengths.Length]; - var len = new int[lengths.Length]; - for (var s = 0; s < lengths.Length; s++) { var l = lengths[s]; if (l == 0) continue; code[s] = next[l]++; len[s] = l; } - return (code, len); - } - - // ── length-limited Huffman code lengths from frequencies ─────────────────── - private static byte[] HuffmanLengths(int[] freq, int maxLen) { - var n = freq.Length; - var lengths = new byte[n]; - var nodes = new System.Collections.Generic.List<(long F, int Sym, int Left, int Right)>(); - for (var s = 0; s < n; s++) if (freq[s] > 0) nodes.Add((freq[s], s, -1, -1)); - if (nodes.Count == 0) return lengths; - if (nodes.Count == 1) { lengths[nodes[0].Sym] = 1; FixKraft(lengths, maxLen); return lengths; } - - var all = new System.Collections.Generic.List<(long F, int Sym, int Left, int Right)>(nodes); - var live = new System.Collections.Generic.List(); - for (var i = 0; i < all.Count; i++) live.Add(i); - while (live.Count > 1) { - live.Sort((a, b) => all[a].F.CompareTo(all[b].F)); - var a = live[0]; var b = live[1]; - live.RemoveRange(0, 2); - all.Add((all[a].F + all[b].F, -1, a, b)); - live.Add(all.Count - 1); - } - AssignDepth(all, live[0], 0, lengths); - FixKraft(lengths, maxLen); - return lengths; - } - - private static void AssignDepth(System.Collections.Generic.List<(long F, int Sym, int Left, int Right)> all, int node, int depth, byte[] lengths) { - var nd = all[node]; - if (nd.Sym >= 0) { lengths[nd.Sym] = (byte)Math.Max(1, depth); return; } - AssignDepth(all, nd.Left, depth + 1, lengths); - AssignDepth(all, nd.Right, depth + 1, lengths); - } - - // Clamp to maxLen and repair the Kraft sum so the code is complete. - private static void FixKraft(byte[] lengths, int maxLen) { - for (var i = 0; i < lengths.Length; i++) if (lengths[i] > maxLen) lengths[i] = (byte)maxLen; - // Kraft sum in units of 2^-maxLen. - long total = 0; - foreach (var l in lengths) if (l > 0) total += 1L << (maxLen - l); - var full = 1L << maxLen; - // Over-full: lengthen some short codes. - while (total > full) { - for (var i = 0; i < lengths.Length && total > full; i++) { - if (lengths[i] > 0 && lengths[i] < maxLen) { total -= 1L << (maxLen - lengths[i]); lengths[i]++; total += 1L << (maxLen - lengths[i]); } - } - } - // Under-full: shorten codes to fill the space (keeps it complete). - var guard = 0; - while (total < full && guard++ < 100000) { - for (var i = 0; i < lengths.Length && total < full; i++) { - if (lengths[i] > 1) { - var delta = 1L << (maxLen - lengths[i]); - if (total + delta <= full) { total += delta; lengths[i]--; } // shortening adds delta again - } - } - if (guard > 99999) break; - } - } + private const int Signature = 0x0081; + private const int HelperSymbols = 22; // code lengths 0..15, run tokens 16..21 + private const int MainSymbols = 0x150; // literals 0..255, tokens 1..80 + private const int MaxCodeLength = 15; + private const int FirstToken = 0x100 - 1; // main symbol of token t is FirstToken + t + private const int FirstLongToken = 64; + private const int EndToken = 80; + private const int ShortMaxLength = 5; + private const int LongMinLength = 6; + private const int LongTokenMaxLength = 21; + private const int MaxMatchLength = LongTokenMaxLength + 0xF + 0xFF + 0xFFF + 0xFFFF; // every extension saturated - public static byte[] Encode(ReadOnlySpan data) { - // table2: Huffman over 0x150 symbols — 256 literals + the end token 0x14F - // (sd4_decomp reads an explicit end token when the output fills). A complete - // code needs >= 2 symbols, so empty/uniform clusters get a nominal literal. - var freq = new int[0x150]; - foreach (var b in data) freq[b]++; - freq[0x14F] = Math.Max(1, freq[0x14F]); // end symbol always present - var nz = 0; foreach (var f in freq) if (f > 0) nz++; - for (var i = 0; i < 256 && nz < 2; i++) if (freq[i] == 0) { freq[i] = 1; nz++; } - var clen = HuffmanLengths(freq, MaxLen); // 0x150 code-lengths - var bmax2 = 0; foreach (var l in clen) if (l > bmax2) bmax2 = l; - if (bmax2 == 0) bmax2 = 1; - - // table1: Huffman over the frequency of each length-VALUE (0..bmax2) in clen. - var lvFreq = new int[0x16]; - foreach (var v in clen) lvFreq[v]++; - var l1 = HuffmanLengths(lvFreq, 7); // table1 bn capped at 7 by the decoder - var bmax1 = 0; foreach (var l in l1) if (l > bmax1) bmax1 = l; - if (bmax1 == 0) bmax1 = 1; - - var (t1code, t1len) = Canonical(l1); - var (t2code, t2len) = Canonical(clen); - - var w = new BitWriter(); - w.Put(0x81, 16); // SD-4 magic - w.Put(0, 3); // start = rdn(3)+1 = 1 - w.Put(bmax2, 5); // ie = bmax2 - w.Put(l1[0], 4); // ca[0] = bmax1 field (length-value 0) - for (var i = 1; i <= bmax2; i++) w.Put(l1[i], 4); // ca[1..bmax2] - for (var i = 0x10; i <= 0x15; i++) w.Put(0, 4); // RLE tokens unused - // emit the 0x150 code-lengths as table1 length-value codes. - foreach (var v in clen) w.Put(t1code[v], t1len[v]); - // emit data as table2 literal codes, then the end token (read by sd4_decomp - // via len_150 bits once the output is full). - foreach (var b in data) w.Put(t2code[b], t2len[b]); - w.Put(t2code[0x14F], t2len[0x14F]); - return w.ToBytes(); - } - - // ── decoding (docs/CVF-CODECS-ON-DISK.md, "SD-4") ────────────────────────── - - private const int HelperSymbols = 0x16; // code lengths 0..15 + run tokens 1..6 - private const int MainSymbols = 0x150; // literals 0..255 + tokens 1..0x50 - private const int EndToken = 0x50; - private const int FirstLongToken = 0x40; - - // Short-match classes: token t (1..0x3F) is class (t-1)/3 with length 3 + (t-1)%3; the offset is - // ShortBase[class] plus ShortBits[class] extra bits. - private static readonly int[] ShortBits = [0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9]; + // Short-match distance classes (token t = 1..63: class (t-1)/3, length 3 + (t-1)%3). private static readonly int[] ShortBase = [1, 2, 3, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 192, 256, 384, 512, 768, 1024, 1536]; + private static readonly int[] ShortBits = [0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9]; - // Long-match offset classes, chosen by a 2-bit selector. + // Long-match distance classes, picked by a 2-bit selector. + private static readonly int[] LongBase = [1, 33, 161, 673]; private static readonly int[] LongBits = [5, 7, 9, 11]; - private static readonly int[] LongBase = [0x1, 0x21, 0xA1, 0x2A1]; - // Escalating length extensions of a long match whose token says 21. + // A long match whose token says 21 carries these extensions, each taken only while the previous one saturated. private static readonly int[] LengthExtensionBits = [4, 8, 12, 16]; + private static readonly int ShortMaxDistance = ShortBase[^1] + (1 << ShortBits[^1]) - 1; // 2047 + private static readonly int LongMaxDistance = LongBase[^1] + (1 << LongBits[^1]) - 1; // 2720 + + // ── decoding ──────────────────────────────────────────────────────────── + /// /// Decodes one SD-4 cluster payload into bytes. A stream that ends (end /// token) before the cluster is full leaves the tail zero. /// - /// Bad magic, an inconsistent code table, or a match that reaches before the cluster start. + /// Bad signature, an inconsistent code table, or a match that reaches before the cluster start. public static byte[] Decode(byte[] payload, int inLen, int outLen) { - var r = new BitReader(payload, inLen); - if (r.ReadN(16) != 0x81) throw new InvalidDataException("SD-4: bad magic."); - - // Helper-table code lengths: symbol 0, symbols first..last, then the six run tokens. - var first = r.ReadN(3) + 1; - var last = r.ReadN(5); - if (first > last || last > 15) throw new InvalidDataException("SD-4: helper table range is invalid."); - var helperLengths = new byte[HelperSymbols]; - helperLengths[0] = (byte)r.ReadN(4); - for (var i = first; i <= last; ++i) helperLengths[i] = (byte)r.ReadN(4); - for (var i = 0x10; i < HelperSymbols; ++i) helperLengths[i] = (byte)r.ReadN(4); - var helper = new CanonicalDecoder(helperLengths); - - var mainLengths = new byte[MainSymbols]; - for (var n = 0; n < MainSymbols;) { - var symbol = helper.Decode(r); - if (symbol < 0x10) { mainLengths[n++] = (byte)symbol; continue; } - if (n == 0 && symbol >= 0x13) throw new InvalidDataException("SD-4: length repeat with no previous length."); - var (value, count) = (symbol - 0x0F) switch { - 1 => ((byte)0, 2), - 2 => ((byte)0, 3), - 3 => ((byte)0, ZeroRunLength(r)), - 4 => (mainLengths[n - 1], 2), - 5 => (mainLengths[n - 1], 3), - _ => (mainLengths[n - 1], RepeatRunLength(r)), - }; - count = Math.Min(count, MainSymbols - n); - mainLengths.AsSpan(n, count).Fill(value); - n += count; - } - var main = new CanonicalDecoder(mainLengths); + var reader = new MsbWordReader(payload.AsSpan(0, Math.Min(inLen, payload.Length))); + if (reader.Read(16) != Signature) throw new InvalidDataException("SD-4: bad signature."); + var main = new CanonicalDecoder(ReadMainLengths(ref reader)); var output = new byte[outLen]; - var p = 0; - while (p < outLen) { - var symbol = main.Decode(r); - if (symbol < 0x100) { output[p++] = (byte)symbol; continue; } + var position = 0; + while (position < outLen) { + var symbol = main.Decode(ref reader); + if (symbol <= 0xFF) { + output[position++] = (byte)symbol; + continue; + } - var token = symbol - 0xFF; + var token = symbol - FirstToken; if (token == EndToken) break; - int length, offset; + int length, distance; if (token < FirstLongToken) { var cls = (token - 1) / 3; length = 3 + (token - 1) % 3; - offset = ShortBase[cls] + r.ReadN(ShortBits[cls]); + distance = ShortBase[cls] + reader.Read(ShortBits[cls]); } else { - var cls = r.ReadN(2); - offset = LongBase[cls] + r.ReadN(LongBits[cls]); - length = token - FirstLongToken + 6; - var saturated = 21; + var selector = reader.Read(2); + distance = LongBase[selector] + reader.Read(LongBits[selector]); + length = token - 58; + var saturated = LongTokenMaxLength; foreach (var bits in LengthExtensionBits) { if (length != saturated) break; - var extra = r.ReadN(bits); - length += extra; + length += reader.Read(bits); saturated += (1 << bits) - 1; } } - if (offset > p) throw new InvalidDataException("SD-4: match reaches before the start of the cluster."); - length = Math.Min(length, outLen - p); // the reference driver clips an overlong final match - for (var k = 0; k < length; ++k, ++p) output[p] = output[p - offset]; + if (distance > position) throw new InvalidDataException("SD-4: match reaches before the start of the cluster."); + length = Math.Min(length, outLen - position); + for (var end = position + length; position < end; ++position) + output[position] = output[position - distance]; } return output; } - // Run token 3: 4 + 3 bits; a saturated 7 continues with 7-bit groups while they read 0x7F. - private static int ZeroRunLength(BitReader r) { - var v = r.ReadN(3); + private static byte[] ReadMainLengths(ref MsbWordReader reader) { + var first = reader.Read(3) + 1; + var last = reader.Read(5); + if (first > last || last > MaxCodeLength) throw new InvalidDataException("SD-4: helper length range is invalid."); + + var helperLengths = new byte[HelperSymbols]; + helperLengths[0] = (byte)reader.Read(4); + for (var i = first; i <= last; ++i) helperLengths[i] = (byte)reader.Read(4); + for (var i = 16; i < HelperSymbols; ++i) helperLengths[i] = (byte)reader.Read(4); + var helper = new CanonicalDecoder(helperLengths); + + var lengths = new byte[MainSymbols]; + for (var n = 0; n < MainSymbols;) { + var symbol = helper.Decode(ref reader); + if (symbol < 16) { + lengths[n++] = (byte)symbol; + continue; + } + if (symbol >= 19 && n == 0) throw new InvalidDataException("SD-4: length repeat with no previous length."); + var (value, count) = symbol switch { + 16 => ((byte)0, 2), + 17 => ((byte)0, 3), + 18 => ((byte)0, ReadZeroRun(ref reader)), + 19 => (lengths[n - 1], 2), + 20 => (lengths[n - 1], 3), + _ => (lengths[n - 1], ReadRepeatRun(ref reader)), + }; + count = Math.Min(count, MainSymbols - n); + lengths.AsSpan(n, count).Fill(value); + n += count; + } + return lengths; + } + + // Helper symbol 18: 4 + 3 bits; a 7 continues with 7-bit groups while they read 0x7F. + private static int ReadZeroRun(ref MsbWordReader reader) { + var v = reader.Read(3); var n = 4 + v; if (v == 7) - do { v = r.ReadN(7); n += v; } while (v == 0x7F); + do n += v = reader.Read(7); while (v == 0x7F); return n; } - // Run token 6: 4 + 3-bit groups, continuing while a group reads 7. - private static int RepeatRunLength(BitReader r) { + // Helper symbol 21: 4 + 3-bit groups, continuing while a group reads 7. + private static int ReadRepeatRun(ref MsbWordReader reader) { var n = 4; int v; - do { v = r.ReadN(3); n += v; } while (v == 7); + do n += v = reader.Read(3); while (v == 7); return n; } + // ── encoding ──────────────────────────────────────────────────────────── + + /// Encodes one cluster with back-references, Huffman-coded behind a run-length coded code table. + /// The cluster. + /// 1 (fastest) and up; deeper match search at higher levels. + public static byte[] Encode(ReadOnlySpan data, int level = 2) + => Encode(data, level, matches: true); + + /// Encodes with back-references switched off when is false — a + /// literals-only stream whose code depth follows the byte statistics alone (used to probe decoders). + internal static byte[] Encode(ReadOnlySpan data, int level, bool matches) { + var symbols = matches ? Parse(data, level) : LiteralsOnly(data); + + var mainFrequencies = new long[MainSymbols]; + foreach (var s in symbols) ++mainFrequencies[s.Symbol]; + EnsureTwoSymbols(mainFrequencies, 0); + var mainLengths = PackageMerge.Lengths(mainFrequencies, MaxCodeLength); + + var runs = RunLengthCode(mainLengths); + var helperFrequencies = new long[HelperSymbols]; + foreach (var r in runs) ++helperFrequencies[r.Symbol]; + EnsureTwoSymbols(helperFrequencies, helperFrequencies[0] == 0 ? 0 : 16); + var helperLengths = PackageMerge.Lengths(helperFrequencies, MaxCodeLength); + + var writer = new MsbWordWriter(); + writer.Write(Signature, 16); + WriteHelperHeader(writer, mainLengths, helperLengths); + + WriteSymbols(writer, runs, helperLengths); + WriteSymbols(writer, symbols, mainLengths); + return writer.ToArray(); + } + + /// A symbol of the main or helper code followed by its extra-bit fields, in stream order. + private readonly record struct CodedSymbol(int Symbol, params (int Value, int Bits)[] Extras); + + private static void WriteSymbols(MsbWordWriter writer, List symbols, int[] lengths) { + var codes = CanonicalCodes(lengths); + foreach (var s in symbols) { + writer.Write(codes[s.Symbol], lengths[s.Symbol]); + foreach (var (value, bits) in s.Extras) writer.Write(value, bits); + } + } + + private static List LiteralsOnly(ReadOnlySpan data) { + var symbols = new List(data.Length + 1); + foreach (var b in data) symbols.Add(new(b)); + symbols.Add(new(FirstToken + EndToken)); + return symbols; + } + + private static List Parse(ReadOnlySpan data, int level) { + var bytes = data.ToArray(); + var finder = new ClusterMatchFinder(bytes, LongMaxDistance, MaxMatchLength, level); + var symbols = new List(bytes.Length / 2 + 2); + + var position = 0; + var pending = Usable(finder.Find(0)); + while (position < bytes.Length) { + if (pending.Length == 0) { + symbols.Add(new(bytes[position], [])); + pending = Usable(finder.Find(++position)); + continue; + } + + var next = position + 1 < bytes.Length ? Usable(finder.Find(position + 1)) : (Length: 0, Distance: 0); + if (next.Length > pending.Length) { + symbols.Add(new(bytes[position], [])); + ++position; + pending = next; + continue; + } + + symbols.Add(MatchSymbol(pending.Length, pending.Distance)); + position += pending.Length; + pending = position < bytes.Length ? Usable(finder.Find(position)) : (Length: 0, Distance: 0); + } + + symbols.Add(new(FirstToken + EndToken, [])); + return symbols; + } + + // Lengths 3..5 only reach as far as the short classes; everything longer may use the long ones. + private static (int Length, int Distance) Usable((int Length, int Distance) match) + => match.Length is > 0 and <= ShortMaxLength && match.Distance > ShortMaxDistance ? (0, 0) : match; + + private static CodedSymbol MatchSymbol(int length, int distance) { + if (length <= ShortMaxLength) { + var cls = ClassOf(ShortBase, ShortBits, distance); + var token = 1 + cls * 3 + (length - 3); + return new(FirstToken + token, [(distance - ShortBase[cls], ShortBits[cls])]); + } + + var selector = ClassOf(LongBase, LongBits, distance); + var extras = new List<(int, int)> { (selector, 2), (distance - LongBase[selector], LongBits[selector]) }; + var longToken = FirstLongToken + Math.Min(length, LongTokenMaxLength) - LongMinLength; + var remaining = length - LongTokenMaxLength; + foreach (var bits in LengthExtensionBits) { + if (remaining < 0) break; + var max = (1 << bits) - 1; + extras.Add((Math.Min(remaining, max), bits)); + remaining -= max; // continues only when this extension saturated + } + return new(FirstToken + longToken, [.. extras]); + } + + private static int ClassOf(int[] bases, int[] bits, int distance) { + for (var c = bases.Length - 1; c >= 0; --c) + if (distance >= bases[c]) + return distance < bases[c] + (1 << bits[c]) ? c : throw new ArgumentOutOfRangeException(nameof(distance)); + throw new ArgumentOutOfRangeException(nameof(distance)); + } + + // A complete prefix code needs at least two symbols; give a spare one a nominal count. + private static void EnsureTwoSymbols(long[] frequencies, int spare) { + var used = frequencies.Count(f => f > 0); + for (var s = spare; used < 2 && s < frequencies.Length; ++s) + if (frequencies[s] == 0) { frequencies[s] = 1; ++used; } + } + /// - /// Canonical MSB-first prefix code: codes ascend with length and, within a length, with symbol - /// number. The code must be complete (Kraft sum exactly one) with lengths of at most 15 bits. + /// Transmits the main code lengths as helper symbols: a length as itself, runs of zeros as tokens + /// 16/17/18, repeats of the previous length as tokens 19/20/21. + /// + private static List RunLengthCode(int[] lengths) { + var runs = new List(); + for (var i = 0; i < lengths.Length;) { + var value = lengths[i]; + var run = 1; + while (i + run < lengths.Length && lengths[i + run] == value) ++run; + i += run; + + if (value == 0) { + runs.Add(run switch { + 1 => new(0), + 2 => new(16), + 3 => new(17), + < 11 => new(18, (run - 4, 3)), + _ => new(18, [(7, 3), .. Groups(run - 11, 7)]), + }); + continue; + } + + runs.Add(new(value)); + var repeats = run - 1; + switch (repeats) { + case 0: break; + case 1: runs.Add(new(value)); break; + case 2: runs.Add(new(19)); break; + case 3: runs.Add(new(20)); break; + default: runs.Add(new(21, [.. Groups(repeats - 4, 3)])); break; + } + } + return runs; + } + + /// + /// Splits into -bit groups where every group but the + /// last is saturated (all ones) and the last is not, which is how the decoder knows to stop. + /// + private static IEnumerable<(int Value, int Bits)> Groups(int total, int bits) { + var saturated = (1 << bits) - 1; + for (; total >= saturated; total -= saturated) yield return (saturated, bits); + yield return (total, bits); + } + + + private static void WriteHelperHeader(MsbWordWriter writer, int[] mainLengths, int[] helperLengths) { + int shortest = MaxCodeLength, longest = 1; + foreach (var l in mainLengths) { + if (l == 0) continue; + shortest = Math.Min(shortest, l); + longest = Math.Max(longest, l); + } + var first = Math.Min(shortest, 8); + writer.Write(first - 1, 3); + writer.Write(longest, 5); + writer.Write(helperLengths[0], 4); + for (var i = first; i <= longest; ++i) writer.Write(helperLengths[i], 4); + for (var i = 16; i < HelperSymbols; ++i) writer.Write(helperLengths[i], 4); + } + + /// Canonical codes: ascending by length, then by symbol, each the previous plus one. + private static int[] CanonicalCodes(int[] lengths) { + var perLength = new int[MaxCodeLength + 1]; + foreach (var l in lengths) ++perLength[l]; + perLength[0] = 0; + var next = new int[MaxCodeLength + 2]; + for (var len = 1; len <= MaxCodeLength; ++len) next[len + 1] = (next[len] + perLength[len]) << 1; + var codes = new int[lengths.Length]; + for (var s = 0; s < lengths.Length; ++s) + if (lengths[s] != 0) codes[s] = next[lengths[s]]++; + return codes; + } + + /// + /// Canonical MSB-first prefix code decoder. The code must be complete (Kraft sum exactly one) with + /// lengths of at most 15 bits. /// private sealed class CanonicalDecoder { - private readonly int[] _count = new int[16]; + private readonly int[] _perLength = new int[MaxCodeLength + 1]; private readonly int[] _symbols; public CanonicalDecoder(ReadOnlySpan lengths) { foreach (var l in lengths) { - if (l > 15) throw new InvalidDataException("SD-4: code length above 15."); - ++this._count[l]; + if (l > MaxCodeLength) throw new InvalidDataException("SD-4: code length above 15."); + ++this._perLength[l]; } - this._count[0] = 0; - var left = 1; - for (var len = 1; len <= 15; ++len) { - left = (left << 1) - this._count[len]; - if (left < 0) throw new InvalidDataException("SD-4: over-subscribed code table."); + this._perLength[0] = 0; + var free = 1; + for (var len = 1; len <= MaxCodeLength; ++len) { + free = (free << 1) - this._perLength[len]; + if (free < 0) throw new InvalidDataException("SD-4: over-subscribed code table."); } - if (left != 0) throw new InvalidDataException("SD-4: incomplete code table."); + if (free != 0) throw new InvalidDataException("SD-4: incomplete code table."); - var offsets = new int[16]; - for (var len = 1; len < 15; ++len) offsets[len + 1] = offsets[len] + this._count[len]; - this._symbols = new int[offsets[15] + this._count[15]]; + var offsets = new int[MaxCodeLength + 2]; + for (var len = 1; len <= MaxCodeLength; ++len) offsets[len + 1] = offsets[len] + this._perLength[len]; + this._symbols = new int[offsets[MaxCodeLength + 1]]; for (var s = 0; s < lengths.Length; ++s) if (lengths[s] != 0) this._symbols[offsets[lengths[s]]++] = s; } - public int Decode(BitReader r) { - int code = 0, firstCode = 0, index = 0; - for (var len = 1; len <= 15; ++len) { - code |= r.ReadN(1); - var count = this._count[len]; - if (code - firstCode < count) return this._symbols[index + code - firstCode]; + public int Decode(ref MsbWordReader reader) { + int code = 0, first = 0, index = 0; + for (var len = 1; len <= MaxCodeLength; ++len) { + code |= reader.Read(1); + var count = this._perLength[len]; + if (code - first < count) return this._symbols[index + code - first]; index += count; - firstCode = (firstCode + count) << 1; + first = (first + count) << 1; code <<= 1; } throw new InvalidDataException("SD-4: undecodable bit sequence."); } } + + /// Reads bits MSB-first from little-endian 16-bit words; zeros past the end. + private ref struct MsbWordReader { + private readonly ReadOnlySpan _data; + private int _next; + private uint _buffer; + private int _count; + + public MsbWordReader(ReadOnlySpan data) => this._data = data; + + public int Read(int count) { + if (count == 0) return 0; + while (this._count < count) { + var lo = this._next < this._data.Length ? this._data[this._next] : 0; + var hi = this._next + 1 < this._data.Length ? this._data[this._next + 1] : 0; + this._next += 2; + this._buffer = this._buffer << 16 | (uint)(lo | hi << 8); + this._count += 16; + } + this._count -= count; + return (int)(this._buffer >> this._count) & ((1 << count) - 1); + } + } + + /// Writes bits MSB-first into little-endian 16-bit words. + private sealed class MsbWordWriter { + private readonly List _bytes = []; + private int _word; + private int _count; + + public void Write(int value, int count) { + for (var i = count - 1; i >= 0; --i) { + this._word = this._word << 1 | (value >> i & 1); + if (++this._count < 16) continue; + this._bytes.Add((byte)this._word); + this._bytes.Add((byte)(this._word >> 8)); + (this._word, this._count) = (0, 0); + } + } + + public byte[] ToArray() { + if (this._count > 0) this.Write(0, 16 - this._count); + return [.. this._bytes]; + } + } } diff --git a/Compression.Core/REFERENCE.md b/Compression.Core/REFERENCE.md index 4aeff3afb..86163b50a 100644 --- a/Compression.Core/REFERENCE.md +++ b/Compression.Core/REFERENCE.md @@ -3318,6 +3318,26 @@ SQX multimedia compressor/decompressor using delta coders (E0-E4) and arithmetic | `Decode` | `static byte[] Decode(byte[] compressed, int originalSize)` | Decompresses multimedia data. | | `Encode` | `static byte[] Encode(ReadOnlySpan data)` | Compresses multimedia data using automatic delta order selection + arithmetic coding. | +### Namespace `Compression.Core.Dictionary.StackerSd4` + +[`StackerSd4BuildingBlock`](#stackersd4buildingblock) + +#### `StackerSd4BuildingBlock` + +Stacker 4's SD-4 codec as a raw primitive: LZ77 over a 2720-byte window, Huffman-coded with a run-length coded code table (`docs/CVF-CODECS-ON-DISK.md`). The building-block wrapper stores the uncompressed size (32-bit little-endian) before one SD-4 stream covering the whole input. + +Implements `IBuildingBlock`. + +| Member | Signature | Summary | +| --- | --- | --- | +| `StackerSd4BuildingBlock` | `StackerSd4BuildingBlock()` | | +| `Description` | `string Description { get; }` | | +| `DisplayName` | `string DisplayName { get; }` | | +| `Family` | `AlgorithmFamily Family { get; }` | | +| `Id` | `string Id { get; }` | | +| `Compress` | `byte[] Compress(ReadOnlySpan data)` | | +| `Decompress` | `byte[] Decompress(ReadOnlySpan data)` | | + ### Namespace `Compression.Core.Dictionary.SuffixTree` [`SuffixTreeBuildingBlock`](#suffixtreebuildingblock) @@ -5627,17 +5647,17 @@ Implements `IEquatable`. #### `CvfLzCodec` -Genuine DoubleSpace/DriveSpace per-cluster compression codec (DS-0-x and JM-0-x), verified byte-exact against the independent dmsdos decoder. The bitstream packs bits LSB-first into little-endian 16-bit words; a cluster payload is a 16-bit magic (`"DS"`=0x5344 / `"MJ"`=0x4D4A) + 16-bit version + an LZ77 token stream terminated by the 0x113f sync. +Per-cluster codecs of the DoubleSpace / DriveSpace / DriveSpace 3 / Stacker compressed volumes, implemented from `docs/CVF-CODECS-ON-DISK.md`. An MSDBL cluster payload starts with a 16-bit magic and a 16-bit version — `"DS"` (DS-0-x), `"JM"` (JM-0-x) or `"SQ"` (SQ-0-0, raw DEFLATE) — followed by the token stream; a Stacker payload carries no method field and is recognised by its first word. | Member | Signature | Summary | | --- | --- | --- | -| `DS_0_0` | `const uint DS_0_0` | | -| `JM_0_0` | `const uint JM_0_0` | | -| `SQ_0_0` | `const uint SQ_0_0` | | -| `Compress` | `static byte[] Compress(ReadOnlySpan data, CvfLzMethod method, int level)` | Compresses one cluster. Returns the payload (4-byte method header + token stream, padded to a 2-byte word), or `null` if it would not be smaller than the raw cluster (caller stores raw instead). | -| `DecompressStacker` | `static byte[] DecompressStacker(byte[] payload, int inLen, int outLen)` | Decompresses a Stacker cluster payload to `outLen` bytes. Stacker records no method per cluster; the payload's first little-endian word selects it — `0x0081` SD-4, `0x5344` ("DS") the DoubleSpace codec, anything else SD-3, whose first token is always a literal and so can never begin with either word. | -| `Decompress` | `static byte[] Decompress(byte[] payload, int inLen, int outLen)` | Decompresses a cluster payload to exactly `outLen` bytes. | -| `Encode` | `static byte[] Encode(ReadOnlySpan data, CvfLzMethod method, int level)` | Encodes a cluster with the given method, always returning the payload (4-byte header + token stream), or `null` for `Stored` / unsupported methods. The caller decides whether the result fits the cluster's sector budget. | +| `DS_0_0` | `const uint DS_0_0` | Method header of a DS-0-0 payload, read as a little-endian 32-bit word. | +| `JM_0_0` | `const uint JM_0_0` | Method header of a JM-0-0 payload, read as a little-endian 32-bit word. | +| `SQ_0_0` | `const uint SQ_0_0` | Method header of an SQ-0-0 payload, read as a little-endian 32-bit word. | +| `Compress` | `static byte[] Compress(ReadOnlySpan data, CvfLzMethod method, int level)` | Compresses one cluster. Returns the payload, or `null` when it would not be smaller than the raw cluster (the caller then stores the cluster raw). | +| `DecompressStacker` | `static byte[] DecompressStacker(byte[] payload, int inLen, int outLen)` | Decompresses a Stacker cluster payload to `outLen` bytes. Stacker records no method per cluster; the payload's first little-endian word selects it — `0x0081` SD-4, `0x5344` ("DS") the DoubleSpace grammar, anything else SD-3, whose first token is always a literal and so can never begin with either word. | +| `Decompress` | `static byte[] Decompress(byte[] payload, int inLen, int outLen)` | Decompresses an MSDBL cluster payload (DS, JM or SQ, told apart by its header) to exactly `outLen` bytes. | +| `Encode` | `static byte[] Encode(ReadOnlySpan data, CvfLzMethod method, int level)` | Encodes a cluster with the given method and always returns the payload, or `null` for `Stored`. The caller decides whether the result fits the cluster's sector budget. `Auto` keeps the smaller of DS and JM. | #### `CvfLzMethod` @@ -5645,13 +5665,13 @@ Per-cluster compression methods of the DoubleSpace/DriveSpace/Stacker CVF family | Value | Numeric | Summary | | --- | --- | --- | -| `Stored` | `0` | | -| `Ds` | `1` | | -| `Jm` | `2` | | -| `Auto` | `3` | | -| `Sq` | `4` | | -| `Sd4` | `5` | | -| `Sd3` | `6` | | +| `Stored` | `0` | No compression; the cluster is stored as is. | +| `Ds` | `1` | DoubleSpace DS-0-x LZ77. | +| `Jm` | `2` | DriveSpace JM-0-x LZ77 (DriveSpace 3 Normal/High). | +| `Auto` | `3` | Per cluster the smaller of `Ds` and `Jm`, both of which every MSDBL volume reads. | +| `Sq` | `4` | DriveSpace 3 SQ-0-0 (Ultra): a raw DEFLATE stream behind the method header. | +| `Sd4` | `5` | Stacker 4 SD-4: Huffman-coded LZ77 behind the header word `0x0081`. | +| `Sd3` | `6` | Stacker 3 SD-3: a bare LZS stream closed by a check byte. | #### `Sd3Codec` @@ -5664,12 +5684,12 @@ Stacker 3 ("SD-3") per-cluster codec, as specified in `docs/CVF-CODECS-ON-DISK.m #### `Sd4Codec` -Genuine Stacker 4 (SD-4, cluster header `0x0081`) per-cluster codec, as specified in `docs/CVF-CODECS-ON-DISK.md`. SD-4 is Huffman-coded LZ77: a helper code (22 symbols: lengths 0..15 and six run tokens) transmits the 0x150 code lengths of the main code, whose symbols are the 256 literals, 63 short-match tokens, 16 long-match tokens and the end token. `Decode` reads the complete format — run tokens, short and long matches with length extensions — and is pinned to streams the dmsdos SD-4 encoder wrote. `Encode` emits an all-literals stream (main code over the cluster's byte frequencies, no match tokens), which the dmsdos driver reads back byte-exact. The bitstream is MSB-first within little-endian 16-bit words; codes are canonical and MSB-first. +Stacker 4 (SD-4, cluster header `0x0081`) per-cluster codec, implemented from `docs/CVF-CODECS-ON-DISK.md`. SD-4 is Huffman-coded LZ77: a helper code (code lengths 0..15 and six run tokens) transmits the 336 code lengths of the main code, whose symbols are the 256 literals, 63 short-match tokens, 16 long-match tokens and the end token. Bits are read MSB-first from little-endian 16-bit words; both codes are canonical and complete. | Member | Signature | Summary | | --- | --- | --- | | `Decode` | `static byte[] Decode(byte[] payload, int inLen, int outLen)` | Decodes one SD-4 cluster payload into `outLen` bytes. A stream that ends (end token) before the cluster is full leaves the tail zero. | -| `Encode` | `static byte[] Encode(ReadOnlySpan data)` | | +| `Encode` | `static byte[] Encode(ReadOnlySpan data, int level = 2)` | Encodes one cluster with back-references, Huffman-coded behind a run-length coded code table. | ### Namespace `Compression.Core.Image` diff --git a/Compression.Tests/Cvf/CvfCodecBoundaryTests.cs b/Compression.Tests/Cvf/CvfCodecBoundaryTests.cs new file mode 100644 index 000000000..ab1057dbd --- /dev/null +++ b/Compression.Tests/Cvf/CvfCodecBoundaryTests.cs @@ -0,0 +1,188 @@ +using Compression.Core.FileSystems.Cvf; + +namespace Compression.Tests.Cvf; + +/// +/// Boundary behaviour of every CVF cluster codec (docs/CVF-CODECS-ON-DISK.md): empty and one-byte +/// clusters, the three cluster sizes the volumes use, incompressible data, and the longest match length +/// and farthest distance each grammar can express — one past each must still round-trip, by falling back +/// to a shorter form. +/// +[TestFixture] +public sealed class CvfCodecBoundaryTests { + + private static readonly CvfLzMethod[] AllMethods = [CvfLzMethod.Ds, CvfLzMethod.Jm, CvfLzMethod.Sq, CvfLzMethod.Sd3, CvfLzMethod.Sd4]; + + private static bool IsStacker(CvfLzMethod method) => method is CvfLzMethod.Sd3 or CvfLzMethod.Sd4; + + private static byte[] RoundTrip(byte[] data, CvfLzMethod method, int level = 2) { + var payload = CvfLzCodec.Encode(data, method, level)!; + return IsStacker(method) + ? CvfLzCodec.DecompressStacker(payload, payload.Length, data.Length) + : CvfLzCodec.Decompress(payload, payload.Length, data.Length); + } + + private static byte[] Random(int length, int seed) { + var data = new byte[length]; + new Random(seed).NextBytes(data); + return data; + } + + [Test, Category("Codec")] + public void GivenAnEmptyCluster_WhenEncodedAndDecoded_ThenNothingComesBack([ValueSource(nameof(AllMethods))] CvfLzMethod method) + => Assert.That(RoundTrip([], method), Is.Empty); + + [Test, Category("Codec")] + public void GivenAOneByteCluster_WhenEncodedAndDecoded_ThenTheByteComesBack( + [ValueSource(nameof(AllMethods))] CvfLzMethod method, [Values(0x00, 0x7F, 0x80, 0xFF)] int value) + => Assert.That(RoundTrip([(byte)value], method), Is.EqualTo(new[] { (byte)value })); + + [Test, Category("Codec")] + public void GivenAClusterOfEachVolumeSize_WhenEncodedAndDecoded_ThenItIsByteExact( + [ValueSource(nameof(AllMethods))] CvfLzMethod method, [Values(512, 8192, 32768)] int size, + [Values("text", "zeros", "random")] string shape) { + var data = shape switch { + "zeros" => new byte[size], + "random" => Random(size, size), + _ => CvfSamples.Get("text-32k")[..size], + }; + + Assert.That(RoundTrip(data, method), Is.EqualTo(data)); + } + + [Test, Category("Codec")] + public void GivenIncompressibleData_WhenCompressed_ThenTheCallerIsToldToStoreIt([Values(CvfLzMethod.Ds, CvfLzMethod.Jm, CvfLzMethod.Sq, CvfLzMethod.Sd4)] CvfLzMethod method) + => Assert.That(CvfLzCodec.Compress(Random(8192, 99), method, 2), Is.Null); + + [Test, Category("Codec")] + public void GivenCompressibleData_WhenCompressed_ThenThePayloadIsSmaller([ValueSource(nameof(AllMethods))] CvfLzMethod method) { + var data = CvfSamples.Get("text-8k"); + + Assert.That(CvfLzCodec.Compress(data, method, 2)!.Length, Is.LessThan(data.Length / 2)); + } + + // DS copies at most 512 bytes per match, JM 513; SD-4's long matches extend at 21, 36, 291 and 4386. + [Test, Category("Codec")] + public void GivenARunAroundEachMaximumMatchLength_WhenEncodedAndDecoded_ThenItIsByteExact( + [ValueSource(nameof(AllMethods))] CvfLzMethod method, + [Values(2, 3, 5, 6, 20, 21, 22, 35, 36, 37, 290, 291, 292, 511, 512, 513, 514, 1025, 4385, 4386, 4387, 8191)] int runLength) { + var data = Random(8192 + 64, runLength); + data.AsSpan(17, runLength).Fill(data[16]); // a literal then a run: one long distance-1 match + + Assert.That(RoundTrip(data, method), Is.EqualTo(data)); + } + + // DS/JM reach 4414 (4415 is the sync marker); SD-4 short matches 2047, long ones 2720; SD-3 2047. + [Test, Category("Codec")] + public void GivenARepeatAtEachDistanceBoundary_WhenEncodedAndDecoded_ThenItIsByteExact( + [ValueSource(nameof(AllMethods))] CvfLzMethod method, + [Values(1, 63, 64, 319, 320, 2047, 2048, 2720, 2721, 4414, 4415, 4416)] int distance, + [Values(3, 5, 6, 40)] int length) { + var data = Random(distance + length + 32, distance * 7 + length); + data.AsSpan(8, length).CopyTo(data.AsSpan(8 + distance)); + + Assert.That(RoundTrip(data, method), Is.EqualTo(data)); + } + + [Test, Category("Codec")] + public void GivenADsOrJmPayload_WhenEncoded_ThenItCarriesTheMethodHeaderAndAWholeNumberOfWords( + [Values(CvfLzMethod.Ds, CvfLzMethod.Jm, CvfLzMethod.Sq)] CvfLzMethod method) { + var payload = CvfLzCodec.Encode(CvfSamples.Get("x86-8k"), method, 1)!; + var expected = method switch { CvfLzMethod.Ds => "DS\0\0"u8.ToArray(), CvfLzMethod.Jm => "JM\0\0"u8.ToArray(), _ => "SQ\0\0"u8.ToArray() }; + + Assert.Multiple(() => { + Assert.That(payload[..4], Is.EqualTo(expected)); + Assert.That(payload.Length % 2, Is.Zero); + }); + } + + [Test, Category("Codec")] + public void GivenAutoMethod_WhenEncoded_ThenTheSmallerOfDsAndJmIsKept() { + var data = CvfSamples.Get("table-8k"); + var ds = CvfLzCodec.Encode(data, CvfLzMethod.Ds, 2)!; + var jm = CvfLzCodec.Encode(data, CvfLzMethod.Jm, 2)!; + + Assert.That(CvfLzCodec.Encode(data, CvfLzMethod.Auto, 2)!.Length, Is.EqualTo(Math.Min(ds.Length, jm.Length))); + } + + [Test, Category("Codec")] + public void GivenStoredMethod_WhenEncoded_ThenThereIsNoPayload() + => Assert.That(CvfLzCodec.Encode(CvfSamples.Get("text-8k"), CvfLzMethod.Stored, 2), Is.Null); + + [Test, Category("Codec")] + public void GivenAPayloadPaddedToWholeSectors_WhenDecoded_ThenThePaddingIsIgnored( + [Values(CvfLzMethod.Ds, CvfLzMethod.Jm, CvfLzMethod.Sq)] CvfLzMethod method) { + var data = CvfSamples.Get("dir-8k"); + var payload = CvfLzCodec.Encode(data, method, 2)!; + var padded = new byte[(payload.Length + 511) / 512 * 512]; + payload.CopyTo(padded, 0); + + Assert.That(CvfLzCodec.Decompress(padded, padded.Length, data.Length), Is.EqualTo(data)); + } + + // A file's last cluster may hold fewer sectors than the cluster: its stream closes with the sync and + // the rest of the cluster reads as zero. + [Test, Category("Codec")] + public void GivenAStreamShorterThanTheCluster_WhenDecoded_ThenTheTailIsZero([Values(CvfLzMethod.Ds, CvfLzMethod.Jm)] CvfLzMethod method) { + var data = CvfSamples.Get("text-8k")[..1536]; + var payload = CvfLzCodec.Encode(data, method, 2)!; + + var cluster = CvfLzCodec.Decompress(payload, payload.Length, 8192); + + Assert.Multiple(() => { + Assert.That(cluster[..1536], Is.EqualTo(data)); + Assert.That(cluster[1536..], Is.All.EqualTo((byte)0)); + }); + } + + [Test, Category("Codec")] + public void GivenATruncatedStream_WhenDecoded_ThenItIsRejected([Values(CvfLzMethod.Ds, CvfLzMethod.Jm)] CvfLzMethod method) { + var payload = CvfLzCodec.Encode(CvfSamples.Get("text-8k"), method, 2)!; + + Assert.That(() => CvfLzCodec.Decompress(payload[..(payload.Length / 2)], payload.Length / 2, 8192), Throws.InstanceOf()); + } + + [Test, Category("Codec")] + public void GivenAnUnknownMethodHeader_WhenDecoded_ThenItIsRejected() + => Assert.That(() => CvfLzCodec.Decompress("XY\0\0\0\0\0\0"u8.ToArray(), 8, 512), Throws.InstanceOf()); + + [Test, Category("Codec")] + public void GivenAPayloadShorterThanItsHeader_WhenDecoded_ThenItIsRejected() + => Assert.That(() => CvfLzCodec.Decompress([0x44, 0x53], 2, 512), Throws.InstanceOf()); + + [Test, Category("Codec")] + public void GivenAMatchBeforeTheClusterStart_WhenDecoded_ThenItIsRejected() { + // DS: "0 0" + distance 5 (LSB-first) then a length code — with nothing decoded yet. + var writer = new LsbBitWriter(); + foreach (var b in "DS\0\0"u8) writer.WriteByte(b); + writer.Write(0b00, 2); writer.Write(5, 6); writer.Write(1, 1); + var payload = writer.ToArray(2); + + Assert.That(() => CvfLzCodec.Decompress(payload, payload.Length, 512), Throws.InstanceOf()); + } + + [Test, Category("Codec")] + public void GivenALengthCodeWithNineLeadingZeros_WhenDecoded_ThenItIsRejected() { + var writer = new LsbBitWriter(); + foreach (var b in "JM\0\0"u8) writer.WriteByte(b); + writer.Write(0x41, 8); // literal 'A' ("0" + 7 bits) + writer.Write(0b01, 2); writer.Write(0, 1); writer.Write(1, 6); // match, distance 1 + writer.Write(0, 9); writer.Write(1, 1); // nine zeros: invalid + var payload = writer.ToArray(2); + + Assert.That(() => CvfLzCodec.Decompress(payload, payload.Length, 512), Throws.InstanceOf()); + } + + [Test, Category("Codec")] + public void GivenDifferentLevels_WhenEncoded_ThenEveryLevelRoundTripsAndHigherIsNoLarger( + [Values(CvfLzMethod.Ds, CvfLzMethod.Jm, CvfLzMethod.Sd4)] CvfLzMethod method) { + var data = CvfSamples.Get("x86-8k"); + var sizes = new List(); + foreach (var level in new[] { 1, 2, 3, 4 }) { + Assert.That(RoundTrip(data, method, level), Is.EqualTo(data), $"level {level}"); + sizes.Add(CvfLzCodec.Encode(data, method, level)!.Length); + } + + Assert.That(sizes[^1], Is.LessThanOrEqualTo(sizes[0])); + } +} diff --git a/Compression.Tests/DoubleSpace/DoubleSpaceHugeVolumeTests.cs b/Compression.Tests/DoubleSpace/DoubleSpaceHugeVolumeTests.cs index 9d53e7c56..15e22d666 100644 --- a/Compression.Tests/DoubleSpace/DoubleSpaceHugeVolumeTests.cs +++ b/Compression.Tests/DoubleSpace/DoubleSpaceHugeVolumeTests.cs @@ -41,8 +41,7 @@ public void Large_Volume_RoundTrips_ViaReader() { [Test, Category("DriverProof")] public void Large_Volume_IsReadByRealDmsdosDriver() { - var build = DmsdosCache.EnsureTools(); - if (build is null) Assert.Ignore("dmsdos unavailable."); + DmsdosOracle.Require(); var files = BigSet(150, 16 * 1024, 2); // ~300 clusters var w = new GenuineCvfWriter { CompressionMethod = CvfLzMethod.Auto, CompressionLevel = 1 }; @@ -52,13 +51,12 @@ public void Large_Volume_IsReadByRealDmsdosDriver() { var cvf = Path.Combine(Path.GetTempPath(), $"cwb-dblhuge-{Guid.NewGuid():N}.cvf"); try { File.WriteAllBytes(cvf, image); - var (_, det) = DmsdosCache.RunTool(DmsdosCache.CvfTest(build!), $"\"{cvf}\" -v"); + var det = Encoding.ASCII.GetBytes(DmsdosOracle.Detect(cvf)); Assert.That(Encoding.ASCII.GetString(det), Does.Contain("drivespace CVF (version 2)")); foreach (var idx in new[] { 0, 50, 100, 149 }) { var (name, data) = files[idx]; - var (exit, raw) = DmsdosCache.RunTool(DmsdosCache.DcRead(build!), $"\"{cvf}\" /{name} raw"); + var (exit, payload) = DmsdosOracle.ReadFile(cvf, name); Assert.That(exit, Is.EqualTo(0), $"dcread {name}"); - var payload = DmsdosCache.PayloadAfterDiagnostics(raw); Assert.That(payload.AsSpan(0, data.Length).ToArray(), Is.EqualTo(data), $"{name} byte-exact"); } } finally { try { File.Delete(cvf); } catch { } } diff --git a/Compression.Tests/DriveSpace3/DriveSpace3GenuineDmsdosTests.cs b/Compression.Tests/DriveSpace3/DriveSpace3GenuineDmsdosTests.cs index e26bee17e..7b322a90d 100644 --- a/Compression.Tests/DriveSpace3/DriveSpace3GenuineDmsdosTests.cs +++ b/Compression.Tests/DriveSpace3/DriveSpace3GenuineDmsdosTests.cs @@ -30,9 +30,7 @@ private static (string Name, byte[] Data)[] SampleFiles() { [TestCase(CvfLzMethod.Sq)] [TestCase(CvfLzMethod.Auto)] public void GenuineDvr3_IsMountedAndReadByteExact_ByRealDmsdosDriver(CvfLzMethod method) { - var build = DmsdosCache.EnsureTools(); - if (build is null) - Assert.Ignore("dmsdos tools unavailable (need Linux + git + cmake + C compiler, or set CWB_DMSDOS_BUILD). Skipping driver-proof gate."); + DmsdosOracle.Require(); var files = SampleFiles(); var writer = new GenuineDvr3Writer { CompressionMethod = method, CompressionLevel = 2 }; @@ -44,7 +42,7 @@ public void GenuineDvr3_IsMountedAndReadByteExact_ByRealDmsdosDriver(CvfLzMethod File.WriteAllBytes(cvfPath, image); // 1. Detection: the real driver must recognise it as DriveSpace 3. - var (_, detOut) = DmsdosCache.RunTool(DmsdosCache.CvfTest(build!), $"\"{cvfPath}\" -v"); + var detOut = Encoding.ASCII.GetBytes(DmsdosOracle.Detect(cvfPath)); var detected = Encoding.ASCII.GetString(detOut); Assert.That(detected, Does.Contain("drivespace 3 CVF"), $"dmsdos did not detect a genuine DriveSpace 3 CVF; got: {detected.Trim()}"); @@ -52,9 +50,8 @@ public void GenuineDvr3_IsMountedAndReadByteExact_ByRealDmsdosDriver(CvfLzMethod // 2. Read-back (write proof): every file must decompress byte-exact // through the real driver — i.e. our writer's output is genuine. foreach (var (name, expected) in files) { - var (exit, raw) = DmsdosCache.RunTool(DmsdosCache.DcRead(build!), $"\"{cvfPath}\" /{name} raw"); + var (exit, payload) = DmsdosOracle.ReadFile(cvfPath, name); Assert.That(exit, Is.EqualTo(0), $"dcread failed for {name}"); - var payload = DmsdosCache.PayloadAfterDiagnostics(raw); Assert.That(payload.Length, Is.GreaterThanOrEqualTo(expected.Length), $"dcread returned too few bytes for {name}"); Assert.That(payload.AsSpan(0, expected.Length).ToArray(), Is.EqualTo(expected), diff --git a/Compression.Tests/DriveSpace3/DriveSpace3GenuineMaintenanceTests.cs b/Compression.Tests/DriveSpace3/DriveSpace3GenuineMaintenanceTests.cs index ddc22e12c..25f7566c3 100644 --- a/Compression.Tests/DriveSpace3/DriveSpace3GenuineMaintenanceTests.cs +++ b/Compression.Tests/DriveSpace3/DriveSpace3GenuineMaintenanceTests.cs @@ -70,8 +70,7 @@ public void Add_Then_Remove_RoundTrips() { [Test, Category("DriverProof")] public void RebuiltImage_StillReadByRealDmsdosDriver() { - var build = DmsdosCache.EnsureTools(); - if (build is null) Assert.Ignore("dmsdos unavailable."); + DmsdosOracle.Require(); var d = new DriveSpace3FormatDescriptor(); using var ms = new MemoryStream(); ms.Write(MakeGenuine()); ms.Position = 0; @@ -81,12 +80,11 @@ public void RebuiltImage_StillReadByRealDmsdosDriver() { var cvf = Path.Combine(Path.GetTempPath(), $"cwb-dvr3reb-{Guid.NewGuid():N}.cvf"); try { File.WriteAllBytes(cvf, img); - var (_, det) = DmsdosCache.RunTool(DmsdosCache.CvfTest(build!), $"\"{cvf}\" -v"); + var det = Encoding.ASCII.GetBytes(DmsdosOracle.Detect(cvf)); Assert.That(Encoding.ASCII.GetString(det), Does.Contain("drivespace 3 CVF")); foreach (var (n, data) in Seed) { - var (exit, raw) = DmsdosCache.RunTool(DmsdosCache.DcRead(build!), $"\"{cvf}\" /{n} raw"); + var (exit, payload) = DmsdosOracle.ReadFile(cvf, n); Assert.That(exit, Is.EqualTo(0), $"dcread {n}"); - var payload = DmsdosCache.PayloadAfterDiagnostics(raw); Assert.That(payload.AsSpan(0, data.Length).ToArray(), Is.EqualTo(data), $"{n} byte-exact after rebuild"); } } finally { try { File.Delete(cvf); } catch { } } diff --git a/Compression.Tests/DriveSpace3/DriveSpace3HugeVolumeTests.cs b/Compression.Tests/DriveSpace3/DriveSpace3HugeVolumeTests.cs index bda8d7eda..01bf2fa30 100644 --- a/Compression.Tests/DriveSpace3/DriveSpace3HugeVolumeTests.cs +++ b/Compression.Tests/DriveSpace3/DriveSpace3HugeVolumeTests.cs @@ -68,8 +68,7 @@ public void Large_Volume_FullReadWrite_AddRemove_ViaDescriptor() { [Test, Category("DriverProof")] public void Large_Volume_IsReadByRealDmsdosDriver() { - var build = DmsdosCache.EnsureTools(); - if (build is null) Assert.Ignore("dmsdos unavailable."); + DmsdosOracle.Require(); var files = BigSet(120, 64 * 1024, 2); // ~240 clusters, mixed var w = new GenuineDvr3Writer { CompressionMethod = CvfLzMethod.Auto, CompressionLevel = 1 }; @@ -79,14 +78,13 @@ public void Large_Volume_IsReadByRealDmsdosDriver() { var cvf = Path.Combine(Path.GetTempPath(), $"cwb-dvr3huge-{Guid.NewGuid():N}.cvf"); try { File.WriteAllBytes(cvf, image); - var (_, det) = DmsdosCache.RunTool(DmsdosCache.CvfTest(build!), $"\"{cvf}\" -v"); + var det = Encoding.ASCII.GetBytes(DmsdosOracle.Detect(cvf)); Assert.That(Encoding.ASCII.GetString(det), Does.Contain("drivespace 3 CVF")); // sample a handful across the volume foreach (var idx in new[] { 0, 37, 80, 119 }) { var (name, data) = files[idx]; - var (exit, raw) = DmsdosCache.RunTool(DmsdosCache.DcRead(build!), $"\"{cvf}\" /{name} raw"); + var (exit, payload) = DmsdosOracle.ReadFile(cvf, name); Assert.That(exit, Is.EqualTo(0), $"dcread {name}"); - var payload = DmsdosCache.PayloadAfterDiagnostics(raw); Assert.That(payload.AsSpan(0, data.Length).ToArray(), Is.EqualTo(data), $"{name} byte-exact"); } } finally { try { File.Delete(cvf); } catch { } } diff --git a/Compression.Tests/ExternalFsInteropTests.cs b/Compression.Tests/ExternalFsInteropTests.cs index 616ce7d8f..772c3b401 100644 --- a/Compression.Tests/ExternalFsInteropTests.cs +++ b/Compression.Tests/ExternalFsInteropTests.cs @@ -2303,9 +2303,7 @@ public void DoubleSpace_OurImage_DmsdosAccepts(Compression.Core.FileSystems.Cvf. // mount it, and read every file back byte-exact. The feature-rich // DoubleSpaceWriter emits a self-consistent (non-genuine) layout; the // driver-genuine emitter is GenuineCvfWriter, proven here. - var build = DmsdosCache.EnsureTools(); - if (build is null) - Assert.Ignore("dmsdos tools unavailable (need Linux + git + cmake + C compiler, or set CWB_DMSDOS_BUILD)."); + DmsdosOracle.Require(); var files = new (string Name, byte[] Data)[] { ("HELLO.TXT", SmallText), @@ -2318,14 +2316,13 @@ public void DoubleSpace_OurImage_DmsdosAccepts(Compression.Core.FileSystems.Cvf. var cvfPath = Path.Combine(this._tmpDir, "dblspace_genuine.cvf"); File.WriteAllBytes(cvfPath, image); - var (_, det) = DmsdosCache.RunTool(DmsdosCache.CvfTest(build!), $"\"{cvfPath}\" -v"); + var det = Encoding.ASCII.GetBytes(DmsdosOracle.Detect(cvfPath)); Assert.That(Encoding.ASCII.GetString(det), Does.Contain("drivespace CVF (version 2)"), "dmsdos did not detect our CVF as a genuine DoubleSpace/DriveSpace v2 volume"); foreach (var (name, expected) in files) { - var (exit, raw) = DmsdosCache.RunTool(DmsdosCache.DcRead(build!), $"\"{cvfPath}\" /{name} raw"); + var (exit, payload) = DmsdosOracle.ReadFile(cvfPath, name); Assert.That(exit, Is.EqualTo(0), $"dcread failed for {name}"); - var payload = DmsdosCache.PayloadAfterDiagnostics(raw); Assert.That(payload.Length, Is.GreaterThanOrEqualTo(expected.Length), $"short read for {name}"); Assert.That(payload.AsSpan(0, expected.Length).ToArray(), Is.EqualTo(expected), $"{name} did not read back byte-exact through the dmsdos driver"); diff --git a/Compression.Tests/Stacker/GenuineStackerCodecOracleTests.cs b/Compression.Tests/Stacker/GenuineStackerCodecOracleTests.cs new file mode 100644 index 000000000..4db2bebb1 --- /dev/null +++ b/Compression.Tests/Stacker/GenuineStackerCodecOracleTests.cs @@ -0,0 +1,187 @@ +using System.Text; +using Compression.Core.FileSystems.Cvf; +using Compression.Tests.Cvf; +using Compression.Tests.Support; +using FileSystem.Stacker; + +namespace Compression.Tests.Stacker; + +/// +/// Stacker cluster codecs against the dmsdos driver as a black box (Linux natively, Windows through +/// WSL): volumes our writer builds must read back byte-exact through dcread and through our own +/// reader. Covers the SD-4 encoder with back-references and the whole-cluster ForceCompress case. +/// +[TestFixture] +public sealed class GenuineStackerCodecOracleTests { + private const int ClusterBytes = 8192; + + private string _dir = null!; + + [SetUp] + public void SetUp() => Directory.CreateDirectory(this._dir = Path.Combine(Path.GetTempPath(), $"cwb-stac-oracle-{Guid.NewGuid():N}")); + + [TearDown] + public void TearDown() { + try { Directory.Delete(this._dir, true); } catch (IOException) { /* best effort */ } + } + + private static byte[] Build(CvfLzMethod method, bool force, params (string Name, byte[] Data)[] files) { + var writer = new GenuineStackerWriter { CompressionMethod = method, CompressionLevel = 3, ForceCompress = force }; + foreach (var (n, d) in files) writer.AddFile(n, d); + return writer.Build(); + } + + private static void AssertOurReaderReads(byte[] image, (string Name, byte[] Data)[] files) { + using var reader = new GenuineStackerReader(new MemoryStream(image)); + foreach (var (name, expected) in files) + Assert.That(reader.Extract(reader.Entries.Single(e => e.Name == name)), Is.EqualTo(expected), $"our reader: {name}"); + } + + private void AssertDmsdosReads(byte[] image, (string Name, byte[] Data)[] files) { + var path = Path.Combine(this._dir, "stac.cvf"); + File.WriteAllBytes(path, image); + Assert.That(DmsdosOracle.Detect(path), Does.Contain("stacker"), "dmsdos does not detect the volume"); + foreach (var (name, expected) in files) { + var (exit, data) = DmsdosOracle.ReadFile(path, name); + Assert.That(exit, Is.Zero, $"dcread failed for {name}"); + Assert.That(data.Length, Is.GreaterThanOrEqualTo(expected.Length), $"dcread returned too few bytes for {name}"); + Assert.That(data[..expected.Length], Is.EqualTo(expected), $"dmsdos: {name}"); + } + } + + /// A cluster whose SD-3 payload needs all 16 sectors: 15 sectors < payload ≤ 16 sectors. + internal static byte[] WholeClusterPayloadData() { + var random = new byte[ClusterBytes]; + new Random(4711).NextBytes(random); + int lo = 0, hi = ClusterBytes; // number of random bytes; the rest repeats one byte + while (lo < hi) { + var mid = (lo + hi) / 2; + var size = Sd3Codec.Encode(Mix(random, mid)).Length; + if (size <= 15 * 512) lo = mid + 1; else hi = mid; + } + var data = Mix(random, lo); + var sectors = (Sd3Codec.Encode(data).Length + 511) / 512; + Assume.That(sectors, Is.EqualTo(16), "could not construct a whole-cluster SD-3 payload"); + return data; + + static byte[] Mix(byte[] random, int count) { + var d = new byte[ClusterBytes]; + random.AsSpan(0, count).CopyTo(d); + return d; + } + } + + [Test, Category("Codec")] + public void GivenForceCompress_WhenACompressedClusterNeedsAllSectors_ThenItIsStoredAndReadsBackExact() { + var data = WholeClusterPayloadData(); + (string, byte[])[] files = [("FULL.BIN", data)]; + + var image = Build(CvfLzMethod.Sd3, force: true, files); + + AssertOurReaderReads(image, files); + } + + [Test, Category("DriverProof")] + public void GivenForceCompress_WhenACompressedClusterNeedsAllSectors_ThenDmsdosReadsItBackExact() { + DmsdosOracle.Require(); + (string, byte[])[] files = [("FULL.BIN", WholeClusterPayloadData())]; + + AssertDmsdosReads(Build(CvfLzMethod.Sd3, force: true, files), files); + } + + private static (string, byte[])[] CodecSamples() { + var text = Encoding.ASCII.GetBytes(string.Concat(Enumerable.Repeat("SD-4 encoder verification line with some repetition.\r\n", 600))); + var runs = new byte[3 * ClusterBytes]; + for (var i = 0; i < runs.Length; ++i) runs[i] = (byte)(i / 5000); // long matches: every extension + return [ + ("TEXT.TXT", text), + ("RUNS.BIN", runs), + ("SAMPTEXT.TXT", CvfSamples.Get("text-8k")), + ("SAMPX86.BIN", CvfSamples.Get("x86-8k")), + ("SAMPTAB.BIN", CvfSamples.Get("table-8k")), + ("SAMPALPH.BIN", CvfSamples.Get("alphabet-8k")), + ("SAMPDIR.BIN", CvfSamples.Get("dir-8k")), + ("ONE.BIN", [0x5A]), + ]; + } + + [Test, Category("DriverProof")] + public void GivenSd4ClustersWithBackReferences_WhenReadByDmsdos_ThenEveryFileIsByteExact([Values(CvfLzMethod.Sd4, CvfLzMethod.Auto)] CvfLzMethod method) { + DmsdosOracle.Require(); + var files = CodecSamples(); + var image = Build(method, force: false, files); + + AssertDmsdosReads(image, files); + AssertOurReaderReads(image, files); + } + + /// + /// One cluster whose byte counts grow just faster than Fibonacci (1, 1, 3, 5, 9, 15, … 1973, each + /// the sum of the two before plus one, so no tie lets a code stay shallow), the rest one more value: + /// an optimal code over them is 16 bits deep, so a literals-only SD-4 stream needs the full 15-bit + /// limit. + /// + internal static byte[] SkewedCluster() { + var data = new List(ClusterBytes); + int a = 1, b = 1; + for (var symbol = 0; symbol < 16; ++symbol, (a, b) = (b, a + b + 1)) + data.AddRange(Enumerable.Repeat((byte)(symbol * 13), a)); + data.AddRange(Enumerable.Repeat((byte)250, ClusterBytes - data.Count)); + var array = data.ToArray(); + new Random(1).Shuffle(array); + return array; + } + + // The 5-bit "last" field after the signature word is the longest main-code length. + private static int LongestMainCode(byte[] payload) => (payload[2] | payload[3] << 8) >> 8 & 0x1F; + + [Test, Category("Codec")] + public void GivenSkewedLiterals_WhenSd4Encoded_ThenTheMainCodeUsesTheFull15BitsAndRoundTrips() { + var data = SkewedCluster(); + var payload = Sd4Codec.Encode(data, level: 2, matches: false); + + Assert.Multiple(() => { + Assert.That(LongestMainCode(payload), Is.EqualTo(15)); + Assert.That(Sd4Codec.Decode(payload, payload.Length, data.Length), Is.EqualTo(data)); + }); + } + + // A stored volume whose one cluster is then rewritten as a compressed run holding the given payload. + private static byte[] VolumeWithClusterPayload(byte[] cluster, byte[] payload) { + var image = Build(CvfLzMethod.Stored, force: false, ("SKEW.BIN", cluster)); + const int amapEntry = (2 + 3) * 512 + 2 * 3; // AMAP area 0 for cluster 2 (FAT start 2) + var sector = image[amapEntry] | image[amapEntry + 1] << 8; + var sectors = (payload.Length + 511) / 512; + Assume.That(sectors, Is.LessThan(16), "payload must be shorter than the cluster"); + Array.Clear(image, sector * 512, 16 * 512); + payload.CopyTo(image, sector * 512); + image[amapEntry + 2] = (byte)(sectors - 1); // flags 0: compressed + return image; + } + + [Test, Category("Codec")] + public void GivenAPatchedSd4Cluster_WhenReadByOurReader_ThenItIsByteExact() { + var data = SkewedCluster(); + (string, byte[])[] files = [("SKEW.BIN", data)]; + + AssertOurReaderReads(VolumeWithClusterPayload(data, Sd4Codec.Encode(data, 2, matches: false)), files); + } + + [Test, Category("DriverProof")] + public void GivenSd4CodesOf15Bits_WhenReadByDmsdos_ThenTheClusterIsByteExact() { + DmsdosOracle.Require(); + var data = SkewedCluster(); + (string, byte[])[] files = [("SKEW.BIN", data)]; + + AssertDmsdosReads(VolumeWithClusterPayload(data, Sd4Codec.Encode(data, 2, matches: false)), files); + } + + [Test, Category("Codec")] + public void GivenSd4Clusters_WhenEncoded_ThenTheyAreWithinTenPercentOfTheDmsdosEncoder( + [Values("text-8k", "x86-8k", "table-8k", "alphabet-8k", "dir-8k")] string sample) { + var ours = Sd4Codec.Encode(CvfSamples.Get(sample), level: 3).Length; + var theirs = CvfReferenceVectorTests.Load($"{sample}.sd4").Length; + + Assert.That(ours, Is.LessThanOrEqualTo(theirs * 11 / 10), $"ours {ours} B, dmsdos {theirs} B"); + } +} diff --git a/Compression.Tests/Stacker/GenuineStackerDmsdosTests.cs b/Compression.Tests/Stacker/GenuineStackerDmsdosTests.cs index efa89ffb3..439273de3 100644 --- a/Compression.Tests/Stacker/GenuineStackerDmsdosTests.cs +++ b/Compression.Tests/Stacker/GenuineStackerDmsdosTests.cs @@ -30,9 +30,7 @@ private static (string Name, byte[] Data)[] SampleFiles() { [TestCase(CvfLzMethod.Sd3)] [TestCase(CvfLzMethod.Auto)] public void GenuineStacker_IsMountedAndReadByteExact_ByRealDmsdosDriver(CvfLzMethod method) { - var build = DmsdosCache.EnsureTools(); - if (build is null) - Assert.Ignore("dmsdos tools unavailable (need Linux + git + cmake + C compiler, or set CWB_DMSDOS_BUILD). Skipping driver-proof gate."); + DmsdosOracle.Require(); var files = SampleFiles(); var writer = new GenuineStackerWriter { CompressionMethod = method, CompressionLevel = 2 }; @@ -44,16 +42,15 @@ public void GenuineStacker_IsMountedAndReadByteExact_ByRealDmsdosDriver(CvfLzMet File.WriteAllBytes(cvfPath, image); // 1. Detection. - var (_, detOut) = DmsdosCache.RunTool(DmsdosCache.CvfTest(build!), $"\"{cvfPath}\" -v"); + var detOut = Encoding.ASCII.GetBytes(DmsdosOracle.Detect(cvfPath)); var detected = Encoding.ASCII.GetString(detOut); Assert.That(detected, Does.Contain("stacker version 3 CVF"), $"dmsdos did not detect a genuine Stacker 3 CVF; got: {detected.Trim()}"); // 2. Write proof: the real driver reads every file back byte-exact. foreach (var (name, expected) in files) { - var (exit, raw) = DmsdosCache.RunTool(DmsdosCache.DcRead(build!), $"\"{cvfPath}\" /{name} raw"); + var (exit, payload) = DmsdosOracle.ReadFile(cvfPath, name); Assert.That(exit, Is.EqualTo(0), $"dcread failed for {name}"); - var payload = DmsdosCache.PayloadAfterDiagnostics(raw); Assert.That(payload.Length, Is.GreaterThanOrEqualTo(expected.Length), $"dcread returned too few bytes for {name}"); Assert.That(payload.AsSpan(0, expected.Length).ToArray(), Is.EqualTo(expected), diff --git a/Compression.Tests/Stacker/StackerFragmentedClusterTests.cs b/Compression.Tests/Stacker/StackerFragmentedClusterTests.cs new file mode 100644 index 000000000..d4b717d8e --- /dev/null +++ b/Compression.Tests/Stacker/StackerFragmentedClusterTests.cs @@ -0,0 +1,151 @@ +using Compression.Core.FileSystems.Cvf; +using Compression.Tests.Cvf; +using Compression.Tests.Support; +using FileSystem.Stacker; + +namespace Compression.Tests.Stacker; + +/// +/// Stacker fragmented clusters (docs/CVF-CODECS-ON-DISK.md, "Stacker fragmented clusters"): a +/// compressed cluster split over the AMAP run and further pieces named by a list in the run's first +/// sector. The layout was established by building volumes and seeing which ones the dmsdos driver reads +/// back; the DriverProof cases here keep that comparison, and our reader must read the same volumes. +/// +[TestFixture] +public sealed class StackerFragmentedClusterTests { + private const int Ss = 512; + private const int AmapEntryOfCluster2 = (2 + 3) * Ss + 2 * 3; // AMAP area 0 (FAT start 2), cluster 2 + + /// How to damage an otherwise valid piece list. + internal enum Damage { None, WrongTotal, FourthByteSet, NoSignature, PiecePastEnd } + + /// + /// A one-file volume whose single cluster is stored fragmented: a first run of + /// sectors holding the list, then pieces of the given sector counts (the + /// last takes whatever is left of the payload). + /// + internal static byte[] Volume(byte[] cluster, byte[] payload, int firstRun, int[] middlePieces, Damage damage = Damage.None) { + var writer = new GenuineStackerWriter { CompressionMethod = CvfLzMethod.Stored }; + writer.AddFile("F.BIN", cluster); + var image = writer.Build(); + var run = image[AmapEntryOfCluster2] | image[AmapEntryOfCluster2 + 1] << 8; + Array.Clear(image, run * Ss, 16 * Ss); + + var pieceCount = middlePieces.Length + 1; + var headerBytes = 4 + 4 * pieceCount; + var inFirst = firstRun * Ss - headerBytes; + var rest = payload.Length - inFirst - middlePieces.Sum() * Ss; + Assume.That(rest, Is.GreaterThan(0), "the last piece must hold part of the payload"); + int[] counts = [.. middlePieces, (rest + Ss - 1) / Ss]; + + var h = run * Ss; + image[h] = damage == Damage.NoSignature ? (byte)0xEC : (byte)0xED; + image[h + 1] = (byte)pieceCount; + var total = counts.Sum() - 1 + (damage == Damage.WrongTotal ? 1 : 0); + image[h + 2] = (byte)total; + image[h + 3] = (byte)(total >> 8); + + payload.AsSpan(0, inFirst).CopyTo(image.AsSpan(h + headerBytes)); + var at = inFirst; + var sector = run + 20; + for (var i = 0; i < counts.Length; ++i, sector += 10) { + var e = h + 4 + 4 * i; + var named = damage == Damage.PiecePastEnd && i == counts.Length - 1 ? image.Length / Ss + 5 : sector; + image[e] = (byte)named; + image[e + 1] = (byte)(named >> 8); + image[e + 2] = (byte)(counts[i] - 1); + image[e + 3] = damage == Damage.FourthByteSet ? (byte)1 : (byte)0; + var take = Math.Min(counts[i] * Ss, payload.Length - at); + payload.AsSpan(at, take).CopyTo(image.AsSpan(sector * Ss)); + at += take; + } + image[AmapEntryOfCluster2 + 2] = (byte)(0x80 | (firstRun - 1)); // bit 7: fragmented; run shorter than the cluster + return image; + } + + private static byte[] Payload(string codec, byte[] data) => codec == "sd3" ? Sd3Codec.Encode(data) : Sd4Codec.Encode(data); + + private static IEnumerable Shapes() { + foreach (var codec in new[] { "sd3", "sd4" }) { + yield return new TestCaseData(codec, 2, Array.Empty()).SetName($"{{m}}({codec}, two pieces)"); + yield return new TestCaseData(codec, 2, new[] { 2 }).SetName($"{{m}}({codec}, three pieces)"); + yield return new TestCaseData(codec, 1, new[] { 1, 1 }).SetName($"{{m}}({codec}, four pieces, one-sector first run)"); + } + } + + private static byte[] Read(byte[] image) { + using var reader = new GenuineStackerReader(new MemoryStream(image)); + return reader.Extract(reader.Entries.Single(e => e.Name == "F.BIN")); + } + + [Test, Category("Codec"), TestCaseSource(nameof(Shapes))] + public void GivenAFragmentedCluster_WhenRead_ThenItIsByteExact(string codec, int firstRun, int[] middle) { + var data = CvfSamples.Get("text-8k"); + + Assert.That(Read(Volume(data, Payload(codec, data), firstRun, middle)), Is.EqualTo(data)); + } + + [Test, Category("DriverProof"), TestCaseSource(nameof(Shapes))] + public void GivenAFragmentedCluster_WhenReadByDmsdos_ThenItIsByteExact(string codec, int firstRun, int[] middle) { + DmsdosOracle.Require(); + var data = CvfSamples.Get("text-8k"); + var path = Path.Combine(Path.GetTempPath(), $"cwb-stac-frag-{Guid.NewGuid():N}.cvf"); + try { + File.WriteAllBytes(path, Volume(data, Payload(codec, data), firstRun, middle)); + var (exit, bytes) = DmsdosOracle.ReadFile(path, "F.BIN"); + + Assert.That(exit, Is.Zero, "dcread failed"); + Assert.That(bytes.Length, Is.GreaterThanOrEqualTo(data.Length)); + Assert.That(bytes[..data.Length], Is.EqualTo(data)); + } finally { + File.Delete(path); + } + } + + // dmsdos refuses a list whose total disagrees with its entries ("sector count mismatch"); so do we. + [Test, Category("DriverProof")] + public void GivenAWrongPieceTotal_WhenReadByDmsdos_ThenItIsRefusedAsByUs() { + DmsdosOracle.Require(); + var data = CvfSamples.Get("text-8k"); + var image = Volume(data, Sd3Codec.Encode(data), 2, [2], Damage.WrongTotal); + var path = Path.Combine(Path.GetTempPath(), $"cwb-stac-frag-{Guid.NewGuid():N}.cvf"); + try { + File.WriteAllBytes(path, image); + var (exit, bytes) = DmsdosOracle.ReadFile(path, "F.BIN"); + + Assert.That(exit != 0 || bytes.Length < data.Length || !bytes[..data.Length].SequenceEqual(data), Is.True, + "dmsdos read a cluster whose piece total is wrong"); + Assert.That(() => Read(image), Throws.InstanceOf()); + } finally { + File.Delete(path); + } + } + + [Test, Category("Codec")] + public void GivenAWrongPieceTotal_WhenRead_ThenItIsRejected() { + var data = CvfSamples.Get("text-8k"); + + Assert.That(() => Read(Volume(data, Sd3Codec.Encode(data), 2, [2], Damage.WrongTotal)), Throws.InstanceOf()); + } + + [Test, Category("Codec")] + public void GivenAPieceEntryWithItsFourthByteSet_WhenRead_ThenItIsRefusedNotGuessed() { + var data = CvfSamples.Get("text-8k"); + + Assert.That(() => Read(Volume(data, Sd3Codec.Encode(data), 2, [2], Damage.FourthByteSet)), Throws.InstanceOf()); + } + + [Test, Category("Codec")] + public void GivenAFragmentedRunWithoutItsSignature_WhenRead_ThenItIsRejected() { + var data = CvfSamples.Get("text-8k"); + + Assert.That(() => Read(Volume(data, Sd3Codec.Encode(data), 2, [2], Damage.NoSignature)), Throws.InstanceOf()); + } + + [Test, Category("Codec")] + public void GivenAPiecePastTheEndOfTheVolume_WhenRead_ThenItIsRejected() { + var data = CvfSamples.Get("text-8k"); + + Assert.That(() => Read(Volume(data, Sd3Codec.Encode(data), 2, [2], Damage.PiecePastEnd)), Throws.InstanceOf()); + } +} diff --git a/Compression.Tests/Stacker/StackerHugeVolumeTests.cs b/Compression.Tests/Stacker/StackerHugeVolumeTests.cs index eeb6b1fd6..1eb1abd9a 100644 --- a/Compression.Tests/Stacker/StackerHugeVolumeTests.cs +++ b/Compression.Tests/Stacker/StackerHugeVolumeTests.cs @@ -40,8 +40,7 @@ public void Above512Clusters_RoundTrips_ViaReader() { [Test, Category("DriverProof")] public void Above512Clusters_IsReadByRealDmsdosDriver() { - var build = DmsdosCache.EnsureTools(); - if (build is null) Assert.Ignore("dmsdos unavailable."); + DmsdosOracle.Require(); var files = BigSet(300, 16 * 1024, 2); // ~600 clusters var w = new GenuineStackerWriter { CompressionMethod = CvfLzMethod.Auto, CompressionLevel = 1 }; @@ -51,13 +50,12 @@ public void Above512Clusters_IsReadByRealDmsdosDriver() { var cvf = Path.Combine(Path.GetTempPath(), $"cwb-stachuge-{Guid.NewGuid():N}.cvf"); try { File.WriteAllBytes(cvf, image); - var (_, det) = DmsdosCache.RunTool(DmsdosCache.CvfTest(build!), $"\"{cvf}\" -v"); + var det = Encoding.ASCII.GetBytes(DmsdosOracle.Detect(cvf)); Assert.That(Encoding.ASCII.GetString(det), Does.Contain("stacker")); foreach (var idx in new[] { 0, 100, 200, 299 }) { var (name, data) = files[idx]; - var (exit, raw) = DmsdosCache.RunTool(DmsdosCache.DcRead(build!), $"\"{cvf}\" /{name} raw"); + var (exit, payload) = DmsdosOracle.ReadFile(cvf, name); Assert.That(exit, Is.EqualTo(0), $"dcread {name}"); - var payload = DmsdosCache.PayloadAfterDiagnostics(raw); Assert.That(payload.AsSpan(0, data.Length).ToArray(), Is.EqualTo(data), $"{name} byte-exact"); } } finally { try { File.Delete(cvf); } catch { } } diff --git a/Compression.Tests/Support/DmsdosOracle.cs b/Compression.Tests/Support/DmsdosOracle.cs new file mode 100644 index 000000000..94a1465e5 --- /dev/null +++ b/Compression.Tests/Support/DmsdosOracle.cs @@ -0,0 +1,53 @@ +namespace Compression.Tests.Support; + +/// +/// The GPL dmsdos tools as a black-box oracle on any host: natively on Linux (built by +/// ), through WSL on Windows (cloned and built once under +/// ~/.cache/cwb-dmsdos inside the distro). Only the tools' behaviour is used — volumes go in, +/// detection text and file contents come out. +/// +internal static class DmsdosOracle { + private static readonly Lazy Build = new(Locate); + + /// Skips the calling test when the tools cannot be had on this machine. + public static void Require() { + if (Build.Value is null) + Assert.Ignore("dmsdos tools unavailable (need Linux or WSL with git + cmake + a C compiler, or CWB_DMSDOS_BUILD)."); + } + + /// What cvftest -v makes of the volume. + public static string Detect(string cvfPath) + => FsInteropToolbox.RunWsl($"'{Build.Value}/cvftest' {FsInteropToolbox.WinToWsl(cvfPath)} -v 2>&1").StdOut; + + /// + /// Reads from the root of the volume with dcread … raw; the bytes are + /// the whole clusters the driver decoded, so callers compare the first file-size bytes. + /// + public static (int Exit, byte[] Data) ReadFile(string cvfPath, string name) { + var outFile = $"{cvfPath}.{name}.out"; + try { + var r = FsInteropToolbox.RunWsl( + $"'{Build.Value}/dcread' {FsInteropToolbox.WinToWsl(cvfPath)} /{name} raw > {FsInteropToolbox.WinToWsl(outFile)} 2>/dev/null"); + var raw = File.Exists(outFile) ? File.ReadAllBytes(outFile) : []; + return (r.ExitCode, DmsdosCache.PayloadAfterDiagnostics(raw)); + } finally { + try { File.Delete(outFile); } catch (IOException) { /* best effort */ } + } + } + + private static string? Locate() { + if (!OperatingSystem.IsWindows()) return DmsdosCache.EnsureTools(); + if (!FsInteropToolbox.WslAvailable) return null; + + const string script = + "set -e; d=$HOME/.cache/cwb-dmsdos; " + + "if [ ! -x $d/build/dcread ] || [ ! -x $d/build/cvftest ]; then " + + "command -v git >/dev/null && command -v cmake >/dev/null && command -v cc >/dev/null; " + + "mkdir -p $d; [ -d $d/src/.git ] || git clone -q --depth 1 " + DmsdosCache.RepoUrl + " $d/src; " + + "cmake -S $d/src -B $d/build -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_VERSION_MINIMUM=3.5 >/dev/null 2>&1; " + + "cmake --build $d/build -j 2 >/dev/null 2>&1; fi; echo $d/build"; + var result = FsInteropToolbox.RunWsl(script); + var lines = result.StdOut.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + return result.ExitCode == 0 && lines.Length > 0 ? lines[^1] : null; + } +} diff --git a/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.DriveSpace3/GenuineDvr3Writer.cs b/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.DriveSpace3/GenuineDvr3Writer.cs index 97ae13e73..bff647676 100644 --- a/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.DriveSpace3/GenuineDvr3Writer.cs +++ b/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.DriveSpace3/GenuineDvr3Writer.cs @@ -66,6 +66,8 @@ public sealed class GenuineDvr3Writer { /// smaller of compressed/stored is chosen per cluster (auto-best). public bool ForceCompress { get; init; } + private static byte[] Smaller(byte[] a, byte[] b) => b.Length < a.Length ? b : a; + /// Adds a file to the root directory of the compressed volume. public void AddFile(string name, byte[] data) { ArgumentNullException.ThrowIfNull(name); @@ -120,7 +122,10 @@ public byte[] Build() { var physCursor = physDataStart; var layout = new List(); foreach (var (cl, full) in clusterFull) { - var comp = CvfLzCodec.Encode(full, this.CompressionMethod, this.CompressionLevel); + // Auto covers every codec a DriveSpace 3 volume reads: DS/JM (the codec's Auto) and SQ. + var comp = this.CompressionMethod == CvfLzMethod.Auto + ? Smaller(CvfLzCodec.Encode(full, CvfLzMethod.Auto, this.CompressionLevel)!, CvfLzCodec.Encode(full, CvfLzMethod.Sq, this.CompressionLevel)!) + : CvfLzCodec.Encode(full, this.CompressionMethod, this.CompressionLevel); var ksize = comp is null ? Spc : (comp.Length + Ss - 1) / Ss; if (comp is not null && ksize <= Spc && (ksize < Spc || this.ForceCompress)) { layout.Add(new ClusterPlan(cl, comp, ksize, true, physCursor)); diff --git a/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Stacker/FORMAT-NOTES.md b/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Stacker/FORMAT-NOTES.md index b1d46e809..2e974f21c 100644 --- a/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Stacker/FORMAT-NOTES.md +++ b/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Stacker/FORMAT-NOTES.md @@ -12,8 +12,9 @@ > [`docs/CVF-CODECS-ON-DISK.md`](../../../docs/CVF-CODECS-ON-DISK.md). The reader > decodes the full SD-4 format (run tokens, short and long matches), pinned to > clusters the dmsdos SD-3/SD-4 compressors wrote (`CvfReferenceVectorTests`); the -> writer's SD-4 output is literals-only. The AMAP flag nibble is honoured: -> fragmented and suballocated clusters are refused, not guessed at. +> writer's SD-4 output uses back-references and run-coded tables, read back by dmsdos. The AMAP flag nibble is honoured: +> fragmented clusters are read (layout measured against dmsdos, see the spec); +> suballocated clusters are refused, not guessed at. > - **The legacy `StackerWriter` (invented `STKMAP01` trailer) is NOT genuine:** > `dmsdos` panics on it. It is retained only for self-round-trip. > diff --git a/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Stacker/GenuineStackerReader.cs b/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Stacker/GenuineStackerReader.cs index 9687298ad..29c716b46 100644 --- a/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Stacker/GenuineStackerReader.cs +++ b/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Stacker/GenuineStackerReader.cs @@ -162,6 +162,48 @@ private ClusterKind Classify(int sectors, int flags) { }; } + // A fragmented cluster (docs/CVF-CODECS-ON-DISK.md, "Stacker fragmented clusters"): the AMAP run is + // the first piece; its first sector starts 0xED, the number n of further pieces, a 16-bit total of + // their sectors minus one, and n entries (16-bit sector, sectors - 1, a byte that must be 0). The + // compressed payload starts right after the entries and continues through the pieces in order. + private byte[] FragmentedPayload(int cluster, int phys, int sectors) { + var first = phys * Ss; + if (first + sectors * Ss > this._data.Length) + throw new InvalidDataException($"Stacker (genuine): fragmented cluster {cluster} lies past the end of the volume."); + if (this._data[first] != 0xED) + throw new InvalidDataException($"Stacker (genuine): fragmented cluster {cluster} lacks its 0xED piece list."); + + var pieces = this._data[first + 1]; + var headerBytes = 4 + 4 * pieces; + if (headerBytes > sectors * Ss) + throw new InvalidDataException($"Stacker (genuine): fragmented cluster {cluster} has a piece list longer than its first run."); + + var listed = new List<(int Sector, int Count)>(pieces); + var sum = 0; + for (var i = 0; i < pieces; ++i) { + var e = first + 4 + 4 * i; + if (this._data[e + 3] != 0) + throw new NotSupportedException($"Stacker (genuine): fragmented cluster {cluster} uses a piece entry whose fourth byte is set; its meaning has not been verified."); + var count = this._data[e + 2] + 1; + listed.Add((this._data[e] | this._data[e + 1] << 8, count)); + sum += count; + } + if (BinaryPrimitives.ReadUInt16LittleEndian(this._data.AsSpan(first + 2)) != sum - 1) + throw new InvalidDataException($"Stacker (genuine): fragmented cluster {cluster} piece total does not match its piece list."); + + var payload = new byte[sectors * Ss - headerBytes + sum * Ss]; + this._data.AsSpan(first + headerBytes, sectors * Ss - headerBytes).CopyTo(payload); + var at = sectors * Ss - headerBytes; + foreach (var (sector, count) in listed) { + var off = (long)sector * Ss; + if (sector == 0 || off + count * Ss > this._data.Length) + throw new InvalidDataException($"Stacker (genuine): fragmented cluster {cluster} names a piece past the end of the volume."); + this._data.AsSpan((int)off, count * Ss).CopyTo(payload.AsSpan(at)); + at += count * Ss; + } + return payload; + } + private int NextCluster(int cluster) { var fatOff = this._fatStart * Ss; if (this._fat16) @@ -205,9 +247,15 @@ public byte[] Extract(StackerEntry entry) { } break; } + case ClusterKind.Fragmented: { + var payload = this.FragmentedPayload(cluster, phys, sectors); + var full = Compression.Core.FileSystems.Cvf.CvfLzCodec.DecompressStacker(payload, payload.Length, clusterBytes); + Array.Copy(full, 0, output, written, want); + break; + } case var kind: - // Fragmented and suballocated runs exist on real volumes, but no sample of either has - // been verified, so they are refused rather than guessed at. + // Suballocated runs exist on real volumes, but their layout has not been verified, so + // they are refused rather than guessed at. throw new NotSupportedException($"Stacker (genuine): cluster {cluster} of '{entry.Name}' is {kind.ToString().ToLowerInvariant()}; reading such clusters is not supported."); } written += want; diff --git a/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Stacker/GenuineStackerWriter.cs b/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Stacker/GenuineStackerWriter.cs index 718afd2d2..fe11ba9dc 100644 --- a/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Stacker/GenuineStackerWriter.cs +++ b/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Stacker/GenuineStackerWriter.cs @@ -74,7 +74,10 @@ public sealed class GenuineStackerWriter { /// Codec effort (search depth). Higher = better ratio, slower. public int CompressionLevel { get; init; } = 1; - /// Keep a compressed cluster even if it does not shrink (auto-best off). + /// Keep a compressed cluster even if it does not shrink (auto-best off). Stacker cannot + /// record a compressed run that fills a whole cluster — such a run reads as stored — so a payload that + /// needs every sector of its cluster is stored raw regardless; any shorter payload is kept compressed + /// either way, which leaves this option without effect on Stacker volumes. public bool ForceCompress { get; init; } private readonly List<(string Name, byte[] Data)> _files = []; @@ -147,7 +150,9 @@ public byte[] Build() { _ => CvfLzCodec.Encode(full, CvfLzMethod.Ds, this.CompressionLevel), }; var ksize = comp is null ? Spc : (comp.Length + Ss - 1) / Ss; - if (comp is not null && ksize < Spc || (comp is not null && this.ForceCompress && ksize <= Spc)) { + // A run as long as the cluster always reads as stored (the AMAP has no flag for a whole-cluster + // compressed run), so a payload needing all Spc sectors is stored raw even under ForceCompress. + if (comp is not null && ksize < Spc) { layout.Add(new ClusterPlan(cl, comp!, ksize, physCursor)); physCursor += ksize; } else { diff --git a/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Stacker/StackerFormatDescriptor.cs b/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Stacker/StackerFormatDescriptor.cs index 63e9ac31f..be615e7ae 100644 --- a/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Stacker/StackerFormatDescriptor.cs +++ b/Hawkynt.FileFormats.FileSystems/FileSystems/FileSystem.Stacker/StackerFormatDescriptor.cs @@ -154,7 +154,8 @@ public sealed class StackerFormatDescriptor : IFormatDescriptor, IArchiveFormatO new FormatOptionDescriptor( Key: "ForceCompress", DisplayName: "Force compression", Kind: FormatOptionKind.Boolean, Default: "false", - Description: "Keep the compressed form even when it does not shrink a cluster.", + Description: "Keep the compressed form even when it does not shrink a cluster. A compressed run " + + "filling a whole cluster cannot be told from a stored one on Stacker, so such a cluster is still stored.", DependsOn: "Compatibility=Genuine"), ]; diff --git a/Hawkynt.FileFormats.FileSystems/REFERENCE.md b/Hawkynt.FileFormats.FileSystems/REFERENCE.md index 446cc52d5..a53eb4bf9 100644 --- a/Hawkynt.FileFormats.FileSystems/REFERENCE.md +++ b/Hawkynt.FileFormats.FileSystems/REFERENCE.md @@ -12929,7 +12929,7 @@ Builds a genuine Stac Electronics STACVOL — the on-disk shape a real Stacker d | `GenuineStackerWriter` | `GenuineStackerWriter()` | | | `CompressionLevel` | `int CompressionLevel { get; init; }` | Codec effort (search depth). Higher = better ratio, slower. | | `CompressionMethod` | `CvfLzMethod CompressionMethod { get; init; }` | Per-cluster compression: Stored (default), Sd3 (Stacker 3), Sd4 (Stacker 4), Ds, or Auto (smallest of Ds/Sd3/Sd4 per cluster). The Stacker driver tells them apart by the payload's first word — 0x0081 SD-4, 0x5344 ("DS") the DoubleSpace codec, anything else SD-3. JM/SQ map to Ds. | -| `ForceCompress` | `bool ForceCompress { get; init; }` | Keep a compressed cluster even if it does not shrink (auto-best off). | +| `ForceCompress` | `bool ForceCompress { get; init; }` | Keep a compressed cluster even if it does not shrink (auto-best off). Stacker cannot record a compressed run that fills a whole cluster — such a run reads as stored — so a payload that needs every sector of its cluster is stored raw regardless; any shorter payload is kept compressed either way, which leaves this option without effect on Stacker volumes. | | `Timestamp` | `DateTime Timestamp { get; init; }` | Creation/modification timestamp stamped on every file entry. Default (before 1980) leaves the FAT date/time fields zero. | | `Version` | `int Version { get; init; }` | Stacker major version recorded in the decoded superblock (< 410 ⇒ v3). | | `VolumeLabel` | `string VolumeLabel { get; init; }` | Optional inner-volume label (≤11 chars). Empty = no label entry. | diff --git a/docs/CVF-CODECS-ON-DISK.md b/docs/CVF-CODECS-ON-DISK.md index b2c7052ed..dc9a6a93e 100644 --- a/docs/CVF-CODECS-ON-DISK.md +++ b/docs/CVF-CODECS-ON-DISK.md @@ -3,7 +3,8 @@ The per-cluster compression formats of the DOS compressed-volume family: Microsoft DoubleSpace / DriveSpace (DS-0-x, JM-0-x), DriveSpace 3 (JM-0-1 "High", SQ-0-0 "Ultra") and Stac Stacker (SD-3, SD-4). `Compression.Core.FileSystems.Cvf` (`CvfLzCodec`, `Sd3Codec`, -`Sd4Codec`) implements this document; the container layouts (MDFAT, AMAP, BitFAT) live in the +`Sd4Codec`) implements this document — it was written from this document and the reference vectors +alone, with dmsdos run only as a black box; the container layouts (MDFAT, AMAP, BitFAT) live in the `FORMAT-NOTES.md` beside each file-system reader. ## 📚 Provenance @@ -45,8 +46,36 @@ its kind), together with whether the run fills a whole cluster: | 1 | 0 | **fragmented** (first sector starts `0xED`, piece count, 4-byte piece list) | stored (directory) | | 1 | 1 | **suballocated** (signature `0x1234` at the sector end) | suballocated | -Fragmented and suballocated clusters are refused by `GenuineStackerReader` — no sample of either -has been verified. +A compressed cluster can therefore never occupy all of its cluster's sectors: a run that long reads +as stored whatever its content (the table above). A writer whose +compressed payload needs every sector stores the cluster raw instead. + +### Stacker fragmented clusters + +Measured with dmsdos as a black box: volumes were built with candidate layouts and kept only where +`dcread` returned the cluster byte-exact (no Stac software or sample volume was available, so this is +"as dmsdos reads it"; `StackerFragmentedClusterTests` keeps the comparison). + +* The AMAP entry (bit 7 set, bit 5 clear) names the **first piece**; its sector count covers that + piece only, header sector included. An AMAP count covering all pieces is rejected. +* The first piece's first sector begins with a header: + + | Offset | Size | Field | + |---|---|---| + | 0 | 1 | `0xED` | + | 1 | 1 | *n*, the number of further pieces | + | 2 | 2 | little-endian: the further pieces' sector counts summed, minus one (dmsdos refuses the cluster when this disagrees with the list) | + | 4 | 4·*n* | one entry per further piece: 16-bit little-endian sector, sector count − 1, one byte that was 0 in every accepted volume | + +* The compressed payload (chosen by its first word as for any Stacker cluster; SD-3 and SD-4 were + verified, a DS payload was not read by dmsdos in this position) starts right after the entries, at byte 4 + 4·*n* of the first piece, and continues through the + pieces in list order. A stored payload is not possible: dmsdos decompresses fragmented clusters. +* Accepted: one to three further pieces, first pieces of one and two sectors, SD-3 and SD-4 payloads. + Not established: the meaning of the entries' fourth byte (perhaps the sector's high bits on volumes + past 65535 sectors) — `GenuineStackerReader` refuses an entry where it is non-zero. + +Suballocated clusters (bit 7 and bit 5 set) are refused by `GenuineStackerReader`; their layout has +not been measured. ## 🔢 DS-0-x and JM-0-x @@ -85,6 +114,17 @@ decoder does not require the final sync. Copies may overlap (distance shorter than length) and are performed byte by byte. +**Measured: one sync per sector.** In all 16 DS/JM reference streams (8 KiB and 32 KiB clusters) +the dmsdos compressors emit a sync after every 512 decoded bytes, not only after the last one, and +no match runs across a 512-byte boundary (0 of all matches) — distances do reach back across +boundaries (up to 4414). Our encoder writes the same shape. The decoder accepts syncs anywhere. + +**A cluster shorter than its nominal size** (a file's last cluster, MDFAT decoded size below the +cluster) ends with a sync followed by nothing but zero padding; decoding stops there and the rest of +the cluster reads as zero. A sync with non-zero bits behind it is a mid-stream marker and decoding +continues. Stacker's use of the DS grammar also ends where the stream runs out or reaches its zero +padding (a `0 0` token with distance 0). + ## 🔢 SQ-0-0 Bytes `53 51` ("SQ"), a 16-bit version (0), then a raw **RFC 1951 DEFLATE** stream: the same block @@ -166,6 +206,12 @@ is extended by 4 bits; if the sum is 36, by 8 more; at 291 by 12 more; at 4386 b not needed to recover the data). A match reaching before the cluster start is invalid; a final match longer than the space left is clipped. +**Our encoder** emits short and long matches (all four length extensions), literals and the end +token; it transmits the code lengths with all six run tokens and builds both codes with package-merge +(optimal, length-limited, always complete). Measured: dmsdos reads main codes 15 bits deep +(`GivenSd4CodesOf15Bits_WhenReadByDmsdos_ThenTheClusterIsByteExact`), so the encoder uses the format's +full 15-bit limit for both codes. + ## ✅ Verification * `Compression.Tests/Cvf/ReferenceVectors/` — 32 cluster payloads written by the dmsdos @@ -179,8 +225,6 @@ match longer than the space left is clipped. ## ⚠️ Not covered -* **Our SD-4 encoder emits literals only.** The decoder reads the whole format; an encoder that - emits match and run tokens has not been written. * **DriveSpace 3 fragmented clusters** (an MDFAT flag dmsdos itself describes as guessed) and - **Stacker fragmented / suballocated clusters** are not read. + **Stacker suballocated clusters** are not read: no layout has been measured for either. * No Microsoft or Stac tool has produced or read any of these streams on this machine. diff --git a/docs/screenshots/archive-browser.png b/docs/screenshots/archive-browser.png index de88d7dd4..e3795c0d0 100644 Binary files a/docs/screenshots/archive-browser.png and b/docs/screenshots/archive-browser.png differ diff --git a/screenshots/main-window.png b/screenshots/main-window.png index e3795c0d0..ffd51c2f5 100644 Binary files a/screenshots/main-window.png and b/screenshots/main-window.png differ