diff --git a/Compression.Core/Dictionary/Azo/AzoDecoder.cs b/Compression.Core/Dictionary/Azo/AzoDecoder.cs new file mode 100644 index 000000000..ca208add4 --- /dev/null +++ b/Compression.Core/Dictionary/Azo/AzoDecoder.cs @@ -0,0 +1,450 @@ +// AZO decoding, converted from EggDotNet (https://github.com/akolman/EggDotNet, MIT, Copyright (c) 2026 +// Andrew Kolman). See THIRD-PARTY-NOTICE.EggDotNet.txt. + +using System.Buffers.Binary; + +namespace Compression.Core.Dictionary.Azo; + +/// +/// ESTsoft's AZO compression (EGG method 3): independent blocks, each an LZ77 parse coded with an +/// adaptive binary arithmetic coder — literal and length models predicted from context, a distance +/// history, a small dictionary of recent matches. +/// +/// +/// Stream layout: a 2-byte header ('1', filter), then blocks introduced by three big-endian UInt32 — +/// uncompressed size, compressed size, and their XOR — until a block with a zero size. A block whose +/// compressed size is not at least 8 bytes smaller than its uncompressed size is stored. +/// +public static class AzoDecoder { + private const byte PrivateVersion = 0x31; // '1' + private const uint ReduceMinSize = 8; + + /// Decodes an AZO stream into exactly bytes. + public static byte[] Decompress(ReadOnlySpan data, long expectedSize) { + if (expectedSize > int.MaxValue) + throw new NotSupportedException("AZO members larger than 2 GiB are not supported."); + if (data.Length < 2 || data[0] != PrivateVersion) + throw new InvalidDataException("Missing or invalid AZO header."); + var output = new byte[expectedSize]; + var produced = 0; + var pos = 2; + while (true) { + if (pos + 12 > data.Length) + throw new InvalidDataException("Truncated AZO block header."); + var blockSize = BinaryPrimitives.ReadUInt32BigEndian(data[pos..]); + var compressSize = BinaryPrimitives.ReadUInt32BigEndian(data[(pos + 4)..]); + var check = BinaryPrimitives.ReadUInt32BigEndian(data[(pos + 8)..]); + pos += 12; + if (blockSize < compressSize || (blockSize ^ compressSize) != check) + throw new InvalidDataException("AZO block size check failed."); + if (blockSize == 0 || compressSize == 0) + break; + if (compressSize > data.Length - pos) + throw new InvalidDataException("AZO block overruns the stream."); + if (blockSize > output.Length - produced) + throw new InvalidDataException("AZO stream holds more data than the member declares."); + var block = output.AsSpan(produced, (int)blockSize); + DecodeBlock(data.Slice(pos, (int)compressSize), block); + pos += (int)compressSize; + produced += (int)blockSize; + if (produced == output.Length && pos + 12 > data.Length) + break; // some writers omit the terminating empty block + } + if (produced != output.Length) + throw new InvalidDataException($"AZO stream produced {produced} bytes, the member declares {expectedSize}."); + return output; + } + + private static void DecodeBlock(ReadOnlySpan input, Span output) { + if (input.Length + ReduceMinSize > output.Length) { + if (input.Length != output.Length) + throw new InvalidDataException("Corrupt AZO stored block."); + input.CopyTo(output); + return; + } + var buffer = new byte[output.Length]; + var entropy = new EntropyCode(input.ToArray()); + if (!new BlockCode(buffer).Code(entropy) || entropy.ConsumedBytes != input.Length) + throw new InvalidDataException("Corrupt AZO block."); + buffer.CopyTo(output); + } + + // ── constants ──────────────────────────────────────────────────────────── + + private const uint AlphaSize = 1 << 8; + private const uint MatchLengthCodeSize = 1 << 7; + private const uint MatchDistCodeSize = 1 << 7; + private const uint MatchDistSgap = 16; + private const uint MatchDistGap = 4; + private const uint MatchMinLength = 2; + private const uint MatchMinDist = 1; + private const uint DistanceHistorySize = 1 << 1; + private const uint DictionarySize = 1 << 7; + private const uint DictionaryHistorySize = 2; + private const int AlphaPredictShift = 5; + private const int LengthPredictShift = 4; + private const int BoolStateBits = 8; + + private static readonly uint[] MatchLengthCodeTable = [ + 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 44, 46, 48, 50, 52, 54, 56, 58, 62, 66, 70, 74, 78, + 82, 86, 90, 98, 106, 114, 122, 130, 138, 146, 154, 170, 186, 202, 218, 234, 250, 266, 282, 314, 346, 378, + 410, 442, 474, 506, 538, 602, 666, 730, 794, 858, 922, 986, 1050, 1178, 1306, 1434, 1562, 1690, 1818, 1946, + 2074, 2330, 2586, 2842, 3098, 3354, 3610, 3866, 4122, 4634, 5146, 5658, 6170, 6682, 7194, 7706, 8218, 9242, + 10266, 11290, 12314, 13338, 14362, 15386, 16410, 18458, 20506, 22554, 24602, 26650, 28698, 30746, + ]; + + private static readonly uint[] MatchDistCodeTable = [ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 25, 27, 29, 33, 37, 41, 45, + 53, 61, 69, 77, 93, 109, 125, 141, 173, 205, 237, 269, 333, 397, 461, 525, 653, 781, 909, 1037, 1293, 1549, + 1805, 2061, 2573, 3085, 3597, 4109, 5133, 6157, 7181, 8205, 10253, 12301, 14349, 16397, 20493, 24589, 28685, + 32781, 40973, 49165, 57357, 65549, 81933, 98317, 114701, 131085, 163853, 196621, 229389, 262157, 327693, + 393229, 458765, 524301, 655373, 786445, 917517, 1048589, 1310733, 1572877, 1835021, 2097165, 2621453, + 3145741, 3670029, 4194317, 5242893, 6291469, 7340045, 8388621, 10485773, 12582925, 14680077, 16777229, + 20971533, 25165837, 29360141, 33554445, 41943053, 50331661, 58720269, 67108877, 83886093, 100663309, + 117440525, 134217741, 167772173, 201326605, 234881037, 268435469, 335544333, 402653197, 469762061, + 536870925, 671088653, 805306381, 939524109, + ]; + + private static readonly uint[] MatchLengthExtraBits = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, + 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, + 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 11, 11, 11, 11, 11, 11, 11, 11, + ]; + + private static readonly uint[] MatchDistExtraBits = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, + 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9, 10, 10, 10, 10, 11, 11, 11, 11, 12, + 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, 15, 15, 16, 16, 16, 16, 17, 17, 17, 17, 18, 18, 18, + 18, 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 23, 23, 23, 23, 24, 24, 24, 24, 25, + 25, 25, 25, 26, 26, 26, 26, 27, 27, 27, 27, + ]; + + /// log2 as the reference computes it (double division), so rounding matches its integer casts. + private static double Log2(uint x) => Math.Log(x) / Math.Log(2); + + private static uint MatchDistCode(uint value) { + value -= MatchMinDist; + if (value < MatchDistSgap) + return value; + value -= MatchDistSgap; + var extraBit = (int)Log2(value / MatchDistGap + 1); + return (uint)(MatchDistSgap + extraBit * MatchDistGap + (value - ((1u << extraBit) - 1) * MatchDistGap) / (1u << extraBit)); + } + + // ── bit input and arithmetic decoder ──────────────────────────────────── + + private sealed class BitCode(byte[] input) { + private readonly uint _totalBits = (uint)input.Length * 8; + private uint _index; + private uint _remain = 8; + private uint _read; + + public uint ReadBits => this._read; + + public byte Read8() { + // all callers read whole bytes from a byte boundary (initialisation) + if (this._totalBits < this._read + 8) + return 0; + this._read += 8; + return input[this._index++]; + } + + public bool ReadBit() { + if (this._totalBits == this._read) + return false; + ++this._read; + --this._remain; + if (this._index >= input.Length) + return false; + var v = (input[this._index] & (1u << (int)this._remain)) != 0; + if (this._remain == 0) { + ++this._index; + this._remain = 8; + } + return v; + } + } + + private sealed class EntropyCode { + private const uint Msb = 0x80000000; + private const uint SecondMsb = 0x40000000; + private readonly BitCode _bits; + private uint _low; + private uint _up = uint.MaxValue; + private uint _tag; + + public EntropyCode(byte[] input) { + this._bits = new BitCode(input); + for (var i = 0; i < 4; ++i) + this._tag |= (uint)this._bits.Read8() << ((3 - i) * 8); + } + + public long ConsumedBytes => (this._bits.ReadBits + 7) / 8; + + private void Rescale() { + while ((this._low & Msb) == (this._up & Msb)) { + this._tag = (this._tag << 1) | (this._bits.ReadBit() ? 1u : 0); + this._low <<= 1; + this._up = (this._up << 1) | 1; + } + while ((this._low & SecondMsb) != 0 && (this._up & SecondMsb) == 0) { + this._tag = ((this._tag << 1) | (this._bits.ReadBit() ? 1u : 0)) ^ Msb; + this._low = (this._low << 1) & (Msb - 1); + this._up = (this._up << 1) | 1 | Msb; + } + } + + private uint Step(int totalBit) => this._low == 0 && this._up == uint.MaxValue + ? 1u << (32 - totalBit) + : (this._up - this._low + 1) >> totalBit; + + /// A value of equiprobable bits. + public uint DecodeUniform(uint totalBit) { + var t = this.Step((int)totalBit); + var v = (this._tag - this._low) / t; + this._up = this._low + t * (v + 1) - 1; + this._low += t * v; + this.Rescale(); + return v; + } + + /// One binary decision whose zero side has probability / 2^. + public bool Decode(uint cumCount, int totalBit) { + var t = this.Step(totalBit); + var v = (this._tag - this._low) / t; + if (v >= cumCount) + this._low += t * cumCount; + else + this._up = this._low + t * cumCount - 1; + this.Rescale(); + return v >= cumCount; + } + } + + // ── adaptive models ────────────────────────────────────────────────────── + + private sealed class BoolState { + private const int TotalBit = 12; + private const uint TotalCount = 1u << TotalBit; + private const uint ArrayN = 1u << BoolStateBits; + private readonly uint[] _prob = [.. Enumerable.Repeat(TotalCount / 2, (int)ArrayN)]; + private uint _state; + + public bool Decode(EntropyCode entropy) { + var b = entropy.Decode(this._prob[this._state], TotalBit); + const int shift = TotalBit - 6; + ref var p = ref this._prob[this._state]; + if (!b) p += (TotalCount - p) >> shift; + else p -= p >> shift; + this._state = ((this._state << 1) & (ArrayN - 1)) | (b ? 1u : 0); + return b; + } + } + + private sealed class EntropyBitProb { + private const int TotalBit = 10; + private const uint TotalCount = 1u << TotalBit; + private readonly int _bitN; + private readonly uint[] _prob; + + public EntropyBitProb(uint n) { + this._bitN = (int)Log2(n - 1) + 1; + this._prob = [.. Enumerable.Repeat(TotalCount / 2, 1 << this._bitN)]; + } + + public uint Decode(EntropyCode entropy) { + uint value = 0, pre = 1; + for (var i = this._bitN - 1; i >= 0; --i) { + var v = entropy.Decode(this._prob[pre], TotalBit); + if (v) value |= 1u << i; + pre = (pre << 1) | (v ? 1u : 0); + } + this.Update(value); + return value; + } + + public void Update(uint value) { + uint pre = 1; + for (var i = this._bitN - 1; i >= 0; --i) { + var v = (value >> i) & 1; + const int shift = TotalBit - 6; + ref var p = ref this._prob[pre]; + if (v == 0) p += (TotalCount - p) >> shift; + else p -= p >> shift; + pre = (pre << 1) | v; + } + } + + /// Which of two models gave the higher probability (sign of the comparison). + public int Compare(EntropyBitProb other, uint value) { + uint prob1 = 1, prob2 = 1, pre = 1; + for (var i = this._bitN - 1; i >= 0; --i) { + var p1 = this._prob[pre]; + var p2 = other._prob[pre]; + var v = ((value >> i) & 1) != 0; + if (((prob1 | prob2) & (((1u << TotalBit) - 1) << (32 - TotalBit))) != 0) { + prob1 >>= TotalBit; + prob2 >>= TotalBit; + } + prob1 *= v ? TotalCount - p1 : p1; + prob2 *= v ? TotalCount - p2 : p2; + pre = (pre << 1) | (v ? 1u : 0); + } + return prob1.CompareTo(prob2); + } + } + + private sealed class PredictProb { + private readonly int _shift; + private readonly EntropyBitProb[] _prob1; + private readonly EntropyBitProb[] _prob2; + private readonly int[] _lucky; + + public PredictProb(uint key, uint n, int shift) { + this._shift = shift; + this._prob1 = [.. Enumerable.Range(0, (int)key).Select(_ => new EntropyBitProb(n))]; + this._prob2 = [.. Enumerable.Range(0, (int)key >> shift).Select(_ => new EntropyBitProb(n))]; + this._lucky = new int[key]; + } + + public uint Decode(EntropyCode entropy, uint pre) { + var p1 = this._prob1[pre]; + var p2 = this._prob2[pre >> this._shift]; + uint v; + if (this._lucky[pre] >= 0) { + v = p1.Decode(entropy); + p2.Update(v); + } else { + v = p2.Decode(entropy); + p1.Update(v); + } + var r = p1.Compare(p2, v); + if (r > 0) ++this._lucky[pre]; + else if (r < 0) --this._lucky[pre]; + return v; + } + } + + private sealed class HistoryList { + private readonly uint[] _rep; + private readonly BoolState _state = new(); + private readonly EntropyBitProb _prob; + + public HistoryList(uint init, uint n) { + this._rep = [.. Enumerable.Range(0, (int)n).Select(i => init + (uint)i)]; + this._prob = new EntropyBitProb(n); + } + + public void Add(uint value) => this.Add(value, (uint)this._rep.Length - 1); + + private void Add(uint value, uint delIdx) { + Array.Copy(this._rep, 0, this._rep, 1, delIdx); + this._rep[0] = value; + } + + public bool TryDecode(EntropyCode entropy, out uint value) { + value = 0; + if (!this._state.Decode(entropy)) + return false; + var idx = this._prob.Decode(entropy); + if (idx >= this._rep.Length) + throw new InvalidDataException("Corrupt AZO history index."); + value = this._rep[idx]; + this.Add(value, idx); + return true; + } + } + + private sealed class SymbolCode(uint n, uint historyN) { + private readonly HistoryList _history = new(0, historyN); + private readonly EntropyBitProb _prob = new(n); + + public uint Decode(EntropyCode entropy) { + if (this._history.TryDecode(entropy, out var ret)) + return ret; + ret = this._prob.Decode(entropy); + this._history.Add(ret); + return ret; + } + } + + private sealed class DictionaryTable { + private readonly (uint Position, uint Length)[] _data = [.. Enumerable.Range(0, (int)DictionarySize).Select(i => (0u, MatchMinLength + (uint)i))]; + private readonly BoolState _findState = new(); + private readonly SymbolCode _prob = new(DictionarySize, DictionaryHistorySize); + + public bool TryDecode(EntropyCode entropy, out uint pos, out uint len) { + pos = len = 0; + if (!this._findState.Decode(entropy)) + return false; + var n = this._prob.Decode(entropy); + if (n >= DictionarySize) + return false; + (pos, len) = this._data[n]; + Array.Copy(this._data, 0, this._data, 1, n); + this._data[0] = (pos, len); + return true; + } + + public void Add(uint pos, uint length) { + Array.Copy(this._data, 0, this._data, 1, this._data.Length - 1); + this._data[0] = (pos, length); + } + } + + private sealed class BlockCode(byte[] buffer) { + private readonly BoolState _matchState = new(); + private readonly PredictProb _alpha = new(AlphaSize, AlphaSize, AlphaPredictShift); + private readonly DictionaryTable _dictionary = new(); + private readonly HistoryList _distHistory = new(MatchMinDist, DistanceHistorySize); + private readonly EntropyBitProb _distProb = new(MatchDistCodeSize); + private readonly PredictProb _lenProb = new(MatchDistCodeSize, MatchLengthCodeSize, LengthPredictShift); + + private uint DecodeDistance(EntropyCode entropy) { + if (this._distHistory.TryDecode(entropy, out var dist)) + return dist; + var idx = this._distProb.Decode(entropy); + if (idx >= MatchDistCodeTable.Length) + throw new InvalidDataException("Corrupt AZO distance code."); + dist = MatchDistCodeTable[idx] + entropy.DecodeUniform(MatchDistExtraBits[idx]); + this._distHistory.Add(dist); + return dist; + } + + private uint DecodeLength(EntropyCode entropy, uint distCode) { + if (distCode >= MatchDistCodeSize) + throw new InvalidDataException("Corrupt AZO distance code."); + var lenCode = this._lenProb.Decode(entropy, distCode); + if (lenCode >= MatchLengthCodeTable.Length) + throw new InvalidDataException("Corrupt AZO length code."); + return MatchLengthCodeTable[lenCode] + entropy.DecodeUniform(MatchLengthExtraBits[lenCode]); + } + + public bool Code(EntropyCode entropy) { + buffer[0] = (byte)this._alpha.Decode(entropy, 0); + for (uint i = 1; i < buffer.Length;) { + if (!this._matchState.Decode(entropy)) { + buffer[i] = (byte)this._alpha.Decode(entropy, buffer[i - 1]); + ++i; + continue; + } + uint dist, length; + if (this._dictionary.TryDecode(entropy, out var dictPos, out length)) + dist = i - dictPos; + else { + dist = this.DecodeDistance(entropy); + length = this.DecodeLength(entropy, MatchDistCode(dist)); + this._dictionary.Add(i, length); + } + if (i < dist || i + length > buffer.Length || length == 0) + return false; + for (var k = 0u; k < length; ++k) + buffer[i + k] = buffer[i - dist + k]; + i += length; + } + return true; + } + } +} diff --git a/Compression.Core/Dictionary/Azo/THIRD-PARTY-NOTICE.EggDotNet.txt b/Compression.Core/Dictionary/Azo/THIRD-PARTY-NOTICE.EggDotNet.txt new file mode 100644 index 000000000..5970171eb --- /dev/null +++ b/Compression.Core/Dictionary/Azo/THIRD-PARTY-NOTICE.EggDotNet.txt @@ -0,0 +1,29 @@ +The AZO decoder (Compression.Core/Dictionary/Azo/AzoDecoder.cs) and the EGG encryption code +(Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Egg/EggCrypto.cs: ZIP 2.0, AES and LEA variants and +the LEA block cipher) are converted to C# from EggDotNet; the EGG test archives under +Compression.Tests/Egg/ReferenceVectors come from its test corpus: + + Upstream: https://github.com/akolman/EggDotNet (master, fetched 2026-10-23) + Licence: MIT + +MIT License + +Copyright (c) 2026 Andrew Kolman (akolman) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Compression.Core/README.md b/Compression.Core/README.md index e918da817..e5e126063 100644 --- a/Compression.Core/README.md +++ b/Compression.Core/README.md @@ -205,7 +205,7 @@ Use the concrete version you intend to consume; this document does not predict a -Every public and protected member of all 734 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Compression.Core/REFERENCE.md). +Every public and protected member of all 735 types, generated from the built assembly and its XML documentation, is in [REFERENCE.md](https://github.com/Hawkynt/CompressionWorkbench/blob/main/Compression.Core/REFERENCE.md). diff --git a/Compression.Core/REFERENCE.md b/Compression.Core/REFERENCE.md index 86510a075..4b0872f11 100644 --- a/Compression.Core/REFERENCE.md +++ b/Compression.Core/REFERENCE.md @@ -636,6 +636,18 @@ Decodes ARJ method 4 ("fastest"), the LZ77 variant ARJ uses when asked for speed | --- | --- | --- | | `Decode` | `static byte[] Decode(ReadOnlySpan compressed, int originalSize)` | Decodes `originalSize` bytes of method-4 data. | +### Namespace `Compression.Core.Dictionary.Azo` + +[`AzoDecoder`](#azodecoder) + +#### `AzoDecoder` + +ESTsoft's AZO compression (EGG method 3): independent blocks, each an LZ77 parse coded with an adaptive binary arithmetic coder — literal and length models predicted from context, a distance history, a small dictionary of recent matches. + +| Member | Signature | Summary | +| --- | --- | --- | +| `Decompress` | `static byte[] Decompress(ReadOnlySpan data, long expectedSize)` | Decodes an AZO stream into exactly `expectedSize` bytes. | + ### Namespace `Compression.Core.Dictionary.Balz` [`BalzBuildingBlock`](#balzbuildingblock) diff --git a/Compression.Tests/Compression.Tests.csproj b/Compression.Tests/Compression.Tests.csproj index 023dbbe39..60ee7e957 100644 --- a/Compression.Tests/Compression.Tests.csproj +++ b/Compression.Tests/Compression.Tests.csproj @@ -146,6 +146,10 @@ ZfsVectors.%(Filename)%(Extension) + + + EggVectors.%(Filename)%(Extension) + G722Vectors.%(Filename)%(Extension) diff --git a/Compression.Tests/Egg/EggReferenceTests.cs b/Compression.Tests/Egg/EggReferenceTests.cs new file mode 100644 index 000000000..f5a0e275e --- /dev/null +++ b/Compression.Tests/Egg/EggReferenceTests.cs @@ -0,0 +1,154 @@ +using System.Reflection; +using System.Text; +using FileFormat.Egg; + +namespace Compression.Tests.Egg; + +/// +/// EGG archives ALZip wrote, from EggDotNet's test corpus (MIT). Each data block carries the CRC-32 +/// ALZip computed, which the reader verifies, so a member that decodes is byte-exact; the content +/// assertions are the ones EggDotNet's own tests make. See ReferenceVectors/README.md. +/// +[TestFixture] +public class EggReferenceTests { + private const string Password = "password12345!"; + + private static byte[] Vector(string name) { + using var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream($"EggVectors.{name}") + ?? throw new FileNotFoundException($"Embedded EGG vector '{name}' is missing from the test assembly."); + using var ms = new MemoryStream(); + stream.CopyTo(ms); + return ms.ToArray(); + } + + private static EggReader Open(string name) => new(new MemoryStream(Vector(name))); + + [TestCase("azoshort_txt.egg", "AZO")] + [TestCase("azonumbers.egg", "AZO")] + [TestCase("lzma_simple.egg", "LZMA")] + [Category("ThirdPartyVector")] + public void Given_CompressedMember_When_Extracted_Then_AlZipCrcHolds(string vector, string method) { + using var r = Open(vector); + var files = r.Entries.Where(e => !e.IsDirectory).ToList(); + Assert.That(files.Select(e => e.MethodName), Has.Some.EqualTo(method)); + foreach (var e in files) + Assert.That(r.Extract(e), Has.Length.EqualTo(e.UncompressedSize), e.Name); + } + + [Test, Category("ThirdPartyVector")] + public void Given_LzmaArchive_When_Extracted_Then_SizesMatchEggDotNet() { + using var r = Open("lzma_simple.egg"); + var e = r.Entries.Single(x => x.Name == "lorem_ipsum.txt"); + Assert.That(e.CompressedSize, Is.EqualTo(5723)); + Assert.That(r.Extract(e), Has.Length.EqualTo(34446)); + } + + [TestCase("lorem_long_zipEnc.egg")] + [TestCase("lorem_long_aes128.egg")] + [TestCase("lorem_long_aes256.egg")] + [TestCase("lorem_long_store_lea128.egg")] + [TestCase("lorem_long_store_lea256.egg")] + [Category("ThirdPartyVector")] + public void Given_EncryptedArchive_When_Decrypted_Then_LoremText(string vector) { + using var r = Open(vector); + var e = r.Entries.Single(x => !x.IsDirectory); + Assert.That(e.IsEncrypted, Is.True); + var text = Encoding.UTF8.GetString(r.Extract(e, Password)); + Assert.That(text, Has.Length.EqualTo(15_238)); + Assert.That(text, Does.StartWith("Lorem ipsum dolor sit amet")); + Assert.That(text, Does.EndWith("sed faucibus orci ligula eu nisi.")); + } + + [TestCase("lorem_long_zipEnc.egg")] + [TestCase("lorem_long_aes256.egg")] + [TestCase("lorem_long_store_lea128.egg")] + [Category("EdgeCase")] + public void Given_EncryptedArchive_When_WrongOrNoPassword_Then_Refused(string vector) { + using var r = Open(vector); + var e = r.Entries.Single(x => !x.IsDirectory); + Assert.Throws(() => r.Extract(e)); + Assert.Throws(() => r.Extract(e, "badpassword")); + } + + [Test, Category("ThirdPartyVector")] + public void Given_SolidArchive_When_Extracted_Then_FirstEqualsLast() { + using var r = Open("solid.egg"); + Assert.That(r.IsSolid, Is.True); + var files = r.Entries.Where(e => !e.IsDirectory).ToList(); + Assert.That(r.Extract(files[^1]), Is.EqualTo(r.Extract(files[0]))); + } + + [TestCase("small_kor.egg", "작은.txt")] + [TestCase("small_shift.egg", "小さい.txt")] + [Category("ThirdPartyVector")] + public void Given_CodePageName_When_Read_Then_DecodedAndHelloWorld(string vector, string name) { + System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance); + using var r = Open(vector); + var e = r.Entries.First(); + Assert.That(e.Name, Is.EqualTo(name)); + Assert.That(Encoding.UTF8.GetString(r.Extract(e)), Is.EqualTo("Hello, world!")); + } + + [Test, Category("ThirdPartyVector")] + public void Given_NestedDirectories_When_Listed_Then_DeepFilePresent() { + using var r = Open("directories.egg"); + Assert.That(r.Entries.Select(e => e.Name), Has.Some.EqualTo("dir1/dir2/dir3/test.txt")); + } + + [Test, Category("ThirdPartyVector")] + public void Given_LargeMember_When_Listed_Then_SizesMatchEggDotNet() { + using var r = Open("zeros.egg"); + var e = r.Entries.Single(); + Assert.That(e.CompressedSize, Is.EqualTo(338)); + Assert.That(e.UncompressedSize, Is.EqualTo(197_540_460)); + } + + [TestCase("posix_small.egg")] + [TestCase("globalcomment.egg")] + [Category("ThirdPartyVector")] + public void Given_Archive_When_EveryMemberExtracted_Then_CrcHolds(string vector) { + using var r = Open(vector); + foreach (var e in r.Entries.Where(e => !e.IsDirectory)) + Assert.That(r.Extract(e), Has.Length.EqualTo(e.UncompressedSize), e.Name); + } + + [Test, Category("EdgeCase")] + public void Given_CorruptAzoBlock_When_Extracted_Then_InvalidData() { + var bytes = Vector("azoshort_txt.egg"); + using var probe = new EggReader(new MemoryStream(bytes)); + var e = probe.Entries.Single(); + bytes[bytes.Length - 20] ^= 0x5A; + using var r = new EggReader(new MemoryStream(bytes)); + Assert.Throws(() => r.Extract(r.Entries.Single())); + } + + private static byte[] OursText() => + Encoding.ASCII.GetBytes(string.Concat(Enumerable.Range(0, 300).Select(i => $"EGG line {i % 17}\r\n"))); + + private static byte[] OursRaw() => [.. Enumerable.Range(0, 3000).Select(i => (byte)(i * 7))]; + + // ours.egg is what our writer produced for these three inputs; EggDotNet read it back + // (ReferenceVectors/eggdotnet-listing.txt: both members Deflate, CRC valid, SHA-256 as below). + [Test, Category("ThirdPartyVector")] + public void Given_SameInputs_When_Written_Then_BytesEqualTheEggDotNetAcceptedFixture() { + using var ms = new MemoryStream(); + new EggFormatDescriptor().Create(ms, [ + Compression.Registry.ArchiveInputInfo.InMemory("docs/text.txt", OursText()), + Compression.Registry.ArchiveInputInfo.InMemory("empty.txt", []), + Compression.Registry.ArchiveInputInfo.InMemory("raw.bin", OursRaw()), + ], new Compression.Registry.FormatCreateOptions()); + Assert.That(ms.ToArray(), Is.EqualTo(Vector("ours.egg"))); + } + + [Test, Category("ThirdPartyVector")] + public void Given_OurFixture_When_Read_Then_MatchesEggDotNetListing() { + using var r = Open("ours.egg"); + Assert.That(r.Entries.Select(e => e.Name), Is.EqualTo(new[] { "docs/text.txt", "empty.txt", "raw.bin" })); + var text = r.Extract(r.Entries.Single(e => e.Name == "docs/text.txt")); + Assert.That(Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(text)), + Is.EqualTo("53ff0c178b3267b2dffbdd59979fba67927f8cc7cc60a4eb209b7c06e92096c9")); + Assert.That(text, Is.EqualTo(OursText())); + Assert.That(r.Extract(r.Entries.Single(e => e.Name == "raw.bin")), Is.EqualTo(OursRaw())); + Assert.That(r.Extract(r.Entries.Single(e => e.Name == "empty.txt")), Is.Empty); + } +} diff --git a/Compression.Tests/Egg/EggTests.cs b/Compression.Tests/Egg/EggTests.cs index 7cfc870d5..da22946ac 100644 --- a/Compression.Tests/Egg/EggTests.cs +++ b/Compression.Tests/Egg/EggTests.cs @@ -183,14 +183,14 @@ public void Writer_LongUtf8Filename_UsesExtendedExtraFieldLength() { [Test, Category("HappyPath")] public void UnsupportedMethod_ListsButThrowsOnExtract() { var payload = "ignored"u8.ToArray(); - // Algorithm 4 = LZMA — listed, but extraction must throw rather than fake bytes. - var archive = BuildStoreOrRaw("movie.lzma", payload, payload, algorithm: 4); + // Algorithm 9 is not an EGG method — listed, but extraction must throw rather than fake bytes. + var archive = BuildStoreOrRaw("movie.lzma", payload, payload, algorithm: 9); var d = new EggFormatDescriptor(); using var ms = new MemoryStream(archive); var entries = d.List(ms, null); Assert.That(entries, Has.Count.EqualTo(1)); - Assert.That(entries[0].Method, Is.EqualTo("LZMA")); + Assert.That(entries[0].Method, Is.EqualTo("Method9")); ms.Position = 0; Assert.Throws(() => diff --git a/Compression.Tests/Egg/ReferenceVectors/.gitattributes b/Compression.Tests/Egg/ReferenceVectors/.gitattributes new file mode 100644 index 000000000..7c0f24129 --- /dev/null +++ b/Compression.Tests/Egg/ReferenceVectors/.gitattributes @@ -0,0 +1,2 @@ +# Fixture bytes are the test: never let a checkout rewrite line endings. +*.egg -text diff --git a/Compression.Tests/Egg/ReferenceVectors/README.md b/Compression.Tests/Egg/ReferenceVectors/README.md new file mode 100644 index 000000000..e08121b62 --- /dev/null +++ b/Compression.Tests/Egg/ReferenceVectors/README.md @@ -0,0 +1,24 @@ +# EGG reference vectors + +All files come unmodified from EggDotNet's test corpus (`src/EggDotNet.Tests/test_files/`, +akolman/EggDotNet master, MIT, fetched 2026-10-23); they were written by ALZip. + +| File | Exercises | +| --- | --- | +| `azoshort_txt.egg`, `azonumbers.egg` | AZO compression | +| `lzma_simple.egg` | LZMA (34 446 bytes in 5 723) | +| `solid.egg` | solid mode (first and last member identical) | +| `lorem_long_zipEnc.egg`, `lorem_long_aes128.egg`, `lorem_long_aes256.egg` | ZIP 2.0, AES-128 and AES-256 encryption (password `password12345!`) | +| `lorem_long_store_lea128.egg`, `lorem_long_store_lea256.egg` | LEA-128 and LEA-256 encryption (same password) | +| `small_kor.egg`, `small_shift.egg` | CP949 and Shift-JIS file names | +| `directories.egg` | nested directories | +| `zeros.egg` | a 197 540 460-byte member (listed, not extracted) | +| `posix_small.egg`, `globalcomment.egg` | POSIX file information, archive comment | + +Each data block stores the CRC-32 ALZip computed, and the reader checks it, so every member these +tests extract is byte-exact; the content assertions repeat EggDotNet's own. + +`ours.egg` is our writer's output (`docs/text.txt`, an empty `empty.txt`, `raw.bin`; Deflate). +EggDotNet (same revision, a net10.0 console that opens each member and checks its CRC) read it +back as recorded in `eggdotnet-listing.txt`. The empty member has no block, as ALZip writes it, so +EggDotNet lists it without opening it. diff --git a/Compression.Tests/Egg/ReferenceVectors/azonumbers.egg b/Compression.Tests/Egg/ReferenceVectors/azonumbers.egg new file mode 100644 index 000000000..c5bc3400c Binary files /dev/null and b/Compression.Tests/Egg/ReferenceVectors/azonumbers.egg differ diff --git a/Compression.Tests/Egg/ReferenceVectors/azoshort_txt.egg b/Compression.Tests/Egg/ReferenceVectors/azoshort_txt.egg new file mode 100644 index 000000000..368faf717 Binary files /dev/null and b/Compression.Tests/Egg/ReferenceVectors/azoshort_txt.egg differ diff --git a/Compression.Tests/Egg/ReferenceVectors/directories.egg b/Compression.Tests/Egg/ReferenceVectors/directories.egg new file mode 100644 index 000000000..932788a15 Binary files /dev/null and b/Compression.Tests/Egg/ReferenceVectors/directories.egg differ diff --git a/Compression.Tests/Egg/ReferenceVectors/eggdotnet-listing.txt b/Compression.Tests/Egg/ReferenceVectors/eggdotnet-listing.txt new file mode 100644 index 000000000..e1f5af848 --- /dev/null +++ b/Compression.Tests/Egg/ReferenceVectors/eggdotnet-listing.txt @@ -0,0 +1,3 @@ +ours.egg docs/text.txt 3720 53ff0c178b3267b2dffbdd59979fba67927f8cc7cc60a4eb209b7c06e92096c9 Deflate crc=True +ours.egg empty.txt 0 (no block) +ours.egg raw.bin 3000 7291514d2492fd7ff49e10ba7df95d19d31d199b89d74bcb62cebdee1bc1a498 Deflate crc=True diff --git a/Compression.Tests/Egg/ReferenceVectors/globalcomment.egg b/Compression.Tests/Egg/ReferenceVectors/globalcomment.egg new file mode 100644 index 000000000..33f738783 Binary files /dev/null and b/Compression.Tests/Egg/ReferenceVectors/globalcomment.egg differ diff --git a/Compression.Tests/Egg/ReferenceVectors/lorem_long_aes128.egg b/Compression.Tests/Egg/ReferenceVectors/lorem_long_aes128.egg new file mode 100644 index 000000000..176d9358b Binary files /dev/null and b/Compression.Tests/Egg/ReferenceVectors/lorem_long_aes128.egg differ diff --git a/Compression.Tests/Egg/ReferenceVectors/lorem_long_aes256.egg b/Compression.Tests/Egg/ReferenceVectors/lorem_long_aes256.egg new file mode 100644 index 000000000..4188e5094 Binary files /dev/null and b/Compression.Tests/Egg/ReferenceVectors/lorem_long_aes256.egg differ diff --git a/Compression.Tests/Egg/ReferenceVectors/lorem_long_store_lea128.egg b/Compression.Tests/Egg/ReferenceVectors/lorem_long_store_lea128.egg new file mode 100644 index 000000000..2490ee939 Binary files /dev/null and b/Compression.Tests/Egg/ReferenceVectors/lorem_long_store_lea128.egg differ diff --git a/Compression.Tests/Egg/ReferenceVectors/lorem_long_store_lea256.egg b/Compression.Tests/Egg/ReferenceVectors/lorem_long_store_lea256.egg new file mode 100644 index 000000000..3f0d179be Binary files /dev/null and b/Compression.Tests/Egg/ReferenceVectors/lorem_long_store_lea256.egg differ diff --git a/Compression.Tests/Egg/ReferenceVectors/lorem_long_zipEnc.egg b/Compression.Tests/Egg/ReferenceVectors/lorem_long_zipEnc.egg new file mode 100644 index 000000000..18a110985 Binary files /dev/null and b/Compression.Tests/Egg/ReferenceVectors/lorem_long_zipEnc.egg differ diff --git a/Compression.Tests/Egg/ReferenceVectors/lzma_simple.egg b/Compression.Tests/Egg/ReferenceVectors/lzma_simple.egg new file mode 100644 index 000000000..8fda92d7f Binary files /dev/null and b/Compression.Tests/Egg/ReferenceVectors/lzma_simple.egg differ diff --git a/Compression.Tests/Egg/ReferenceVectors/ours.egg b/Compression.Tests/Egg/ReferenceVectors/ours.egg new file mode 100644 index 000000000..0c19c089e Binary files /dev/null and b/Compression.Tests/Egg/ReferenceVectors/ours.egg differ diff --git a/Compression.Tests/Egg/ReferenceVectors/posix_small.egg b/Compression.Tests/Egg/ReferenceVectors/posix_small.egg new file mode 100644 index 000000000..c17e22755 Binary files /dev/null and b/Compression.Tests/Egg/ReferenceVectors/posix_small.egg differ diff --git a/Compression.Tests/Egg/ReferenceVectors/small_kor.egg b/Compression.Tests/Egg/ReferenceVectors/small_kor.egg new file mode 100644 index 000000000..041a39db7 Binary files /dev/null and b/Compression.Tests/Egg/ReferenceVectors/small_kor.egg differ diff --git a/Compression.Tests/Egg/ReferenceVectors/small_shift.egg b/Compression.Tests/Egg/ReferenceVectors/small_shift.egg new file mode 100644 index 000000000..762dc376f Binary files /dev/null and b/Compression.Tests/Egg/ReferenceVectors/small_shift.egg differ diff --git a/Compression.Tests/Egg/ReferenceVectors/solid.egg b/Compression.Tests/Egg/ReferenceVectors/solid.egg new file mode 100644 index 000000000..bb99ca46e Binary files /dev/null and b/Compression.Tests/Egg/ReferenceVectors/solid.egg differ diff --git a/Compression.Tests/Egg/ReferenceVectors/zeros.egg b/Compression.Tests/Egg/ReferenceVectors/zeros.egg new file mode 100644 index 000000000..78fd775ed Binary files /dev/null and b/Compression.Tests/Egg/ReferenceVectors/zeros.egg differ diff --git a/Compression.Tests/FirmwareHex/FirmwareHexPayloadTests.cs b/Compression.Tests/FirmwareHex/FirmwareHexPayloadTests.cs new file mode 100644 index 000000000..bbbc898af --- /dev/null +++ b/Compression.Tests/FirmwareHex/FirmwareHexPayloadTests.cs @@ -0,0 +1,37 @@ +using Compression.Registry; +using FileFormat.FirmwareHex; + +namespace Compression.Tests.FirmwareHex; + +[TestFixture] +public sealed class FirmwareHexPayloadTests { + [TestCase(false)] + [TestCase(true)] + public void Create_RejectsMultiplePayloadsWithoutWritingOutput(bool emptyFirst) { + var descriptor = new IntelHexFormatDescriptor(); + using var output = new MemoryStream(); + var inputs = new[] { + ArchiveInputInfo.InMemory("first.bin", emptyFirst ? [] : [1, 2, 3]), + ArchiveInputInfo.InMemory("second.bin", [9, 8, 7]), + }; + + Assert.Throws(() => + ((IArchiveCreatable)descriptor).Create(output, inputs, new FormatCreateOptions())); + Assert.That(output.Length, Is.Zero); + } + + [Test] + public void Add_RejectsAnotherPayloadWithoutChangingTheArchive() { + var descriptor = new IntelHexFormatDescriptor(); + using var archive = new MemoryStream(); + ((IArchiveCreatable)descriptor).Create(archive, + [ArchiveInputInfo.InMemory("firmware.bin", [1, 2, 3])], new FormatCreateOptions()); + var original = archive.ToArray(); + archive.Position = 0; + + Assert.Throws(() => ((IArchiveModifiable)descriptor).Add(archive, + [ArchiveInputInfo.InMemory("ADD.BIN", [9, 8, 7])])); + Assert.That(archive.ToArray(), Is.EqualTo(original)); + } + +} diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Egg/EggCrypto.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Egg/EggCrypto.cs new file mode 100644 index 000000000..48c64d897 --- /dev/null +++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Egg/EggCrypto.cs @@ -0,0 +1,233 @@ +// EGG member encryption, converted from EggDotNet (https://github.com/akolman/EggDotNet, MIT, Copyright (c) +// 2026 Andrew Kolman); see THIRD-PARTY-NOTICE.EggDotNet.txt. LEA itself is the Korean standard block +// cipher (TTAS.KO-12.0223, ISO/IEC 29192-2). + +using System.Buffers.Binary; +using System.Numerics; +using System.Security.Cryptography; +using System.Text; + +namespace FileFormat.Egg; + +/// +/// The encryption field of an EGG file header: method 0 ZIP 2.0 ("standard"), 1/2 AES-128/256 and +/// 5/6 LEA-128/256, with the per-method header (salt + password check) and footer (MAC or CRC part). +/// +internal sealed record EggEncryption(byte Method, byte[] Header, byte[] Footer) { + public const byte Standard = 0, Aes128 = 1, Aes256 = 2, Lea128 = 5, Lea256 = 6; + + public static EggEncryption Parse(ReadOnlySpan field) { + if (field.IsEmpty) + throw new InvalidDataException("Empty EGG encryption field."); + var method = field[0]; + var (headerLength, footerLength) = method switch { + Standard => (12, 4), + Aes128 or Lea128 => (10, 10), + Aes256 or Lea256 => (18, 10), + _ => throw new NotSupportedException($"EGG encryption method {method} is unknown."), + }; + if (field.Length < 1 + headerLength + footerLength) + throw new InvalidDataException("Truncated EGG encryption field."); + return new(method, field.Slice(1, headerLength).ToArray(), field.Slice(1 + headerLength, footerLength).ToArray()); + } + + public string MethodName => this.Method switch { + Standard => "ZIP 2.0", + Aes128 => "AES-128", + Aes256 => "AES-256", + Lea128 => "LEA-128", + Lea256 => "LEA-256", + _ => $"method {this.Method}", + }; + + /// Decrypts a member's packed bytes; throws on a wrong password or MAC. + public byte[] Decrypt(byte[] data, string password) => this.Method switch { + Standard => DecryptStandard(data, password), + Aes128 or Aes256 => DecryptWinZipStyle(data, password, this.Method == Aes256 ? 256 : 128, lea: false), + Lea128 or Lea256 => DecryptWinZipStyle(data, password, this.Method == Lea256 ? 256 : 128, lea: true), + _ => throw new NotSupportedException($"EGG encryption method {this.Method} is unknown."), + }; + + // ── ZIP 2.0 ─────────────────────────────────────────────────────────── + + private byte[] DecryptStandard(byte[] data, string password) { + var keys = new ZipKeys(); + foreach (var c in password) + keys.Update((byte)c); + byte last = 0; + foreach (var b in this.Header) { + last = (byte)(b ^ keys.Stream()); + keys.Update(last); + } + if (last != this.Footer[^1]) + throw new InvalidDataException("Wrong EGG password."); + var output = (byte[])data.Clone(); + for (var i = 0; i < output.Length; ++i) { + var p = (byte)(output[i] ^ keys.Stream()); + keys.Update(p); + output[i] = p; + } + return output; + } + + private sealed class ZipKeys { + private static readonly uint[] Table = [.. Enumerable.Range(0, 256).Select(i => { + var c = (uint)i; + for (var j = 0; j < 8; ++j) + c = (c & 1) != 0 ? (c >> 1) ^ 0xEDB88320u : c >> 1; + return c; + })]; + private uint _k0 = 305419896, _k1 = 591751049, _k2 = 878082192; + + public void Update(byte b) { + this._k0 = Table[(this._k0 ^ b) & 0xFF] ^ (this._k0 >> 8); + this._k1 = (this._k1 + (this._k0 & 0xFF)) * 134775813 + 1; + this._k2 = Table[(this._k2 ^ (this._k1 >> 24)) & 0xFF] ^ (this._k2 >> 8); + } + + public byte Stream() { + var t = (this._k2 | 2) & 0xFFFF; + return (byte)((t * (t ^ 1)) >> 8); + } + } + + // ── AES / LEA (WinZip-style key derivation, CTR, HMAC-SHA1) ─────────── + + private byte[] DecryptWinZipStyle(byte[] data, string password, int bits, bool lea) { + var saltLength = bits == 256 ? 16 : 8; + var keyLength = bits / 8; + var macLength = lea ? (bits == 256 ? 32 : 16) : keyLength; + var salt = this.Header.AsSpan(0, saltLength); + var storedCheck = this.Header.AsSpan(saltLength, 2); + var derived = Rfc2898DeriveBytes.Pbkdf2(Encoding.UTF8.GetBytes(password), salt, 1000, HashAlgorithmName.SHA1, keyLength + macLength + 2); + var key = derived.AsSpan(0, keyLength); + var macKey = derived.AsSpan(keyLength, macLength); + var check = derived.AsSpan(keyLength + macLength, 2); + + var valid = lea + ? check.SequenceEqual(storedCheck) + : (short)(check[0] + check[1] * bits) == (short)(storedCheck[0] + storedCheck[1] * bits); + if (!valid) + throw new InvalidDataException("Wrong EGG password."); + + var mac = HMACSHA1.HashData(macKey, data); + if (!mac.AsSpan(0, 10).SequenceEqual(this.Footer)) + throw new InvalidDataException("EGG member failed its authentication code (wrong password or corrupt data)."); + + return lea ? LeaCtr(data, key.ToArray()) : AesCtrLittleEndian(data, key.ToArray()); + } + + /// AES in CTR mode with WinZip's counter: a little-endian block number starting at 1. + private static byte[] AesCtrLittleEndian(byte[] data, byte[] key) { + using var aes = Aes.Create(); + aes.Key = key; + var output = new byte[data.Length]; + var counter = new byte[16]; + var stream = new byte[16]; + ulong block = 1; + for (var pos = 0; pos < data.Length; pos += 16) { + BinaryPrimitives.WriteUInt64LittleEndian(counter, block++); + aes.EncryptEcb(counter, stream, PaddingMode.None); + var n = Math.Min(16, data.Length - pos); + for (var i = 0; i < n; ++i) + output[pos + i] = (byte)(data[pos + i] ^ stream[i]); + } + return output; + } + + /// LEA in CTR mode: zero IV, big-endian counter increment. + private static byte[] LeaCtr(byte[] data, byte[] key) { + var cipher = new Lea(key); + var output = new byte[data.Length]; + var counter = new byte[16]; + var stream = new byte[16]; + for (var pos = 0; pos < data.Length; pos += 16) { + cipher.EncryptBlock(counter, stream); + for (var i = 15; i >= 0; --i) + if (++counter[i] != 0) + break; + var n = Math.Min(16, data.Length - pos); + for (var i = 0; i < n; ++i) + output[pos + i] = (byte)(data[pos + i] ^ stream[i]); + } + return output; + } +} + +/// The LEA block cipher (128-bit blocks; 128-, 192- or 256-bit keys). +internal sealed class Lea { + private static readonly uint[] Delta = [0xc3efe9db, 0x44626b02, 0x79e27c8a, 0x78df30ec, 0x715ea49e, 0xc785da0a, 0xe04ef22a, 0xe5c40957]; + private readonly uint[,] _rk; + private readonly int _rounds; + + private static uint Rol(uint v, int n) => BitOperations.RotateLeft(v, n); + private static uint Ror(uint v, int n) => BitOperations.RotateRight(v, n); + + public Lea(byte[] key) { + if (key.Length is not (16 or 24 or 32)) + throw new ArgumentException("LEA keys are 16, 24 or 32 bytes.", nameof(key)); + this._rounds = (key.Length >> 1) + 16; + this._rk = new uint[this._rounds, 6]; + var t = new uint[8]; + for (var i = 0; i < key.Length / 4; ++i) + t[i] = BinaryPrimitives.ReadUInt32LittleEndian(key.AsSpan(i * 4)); + switch (key.Length) { + case 16: + for (var i = 0; i < 24; ++i) { + var temp = Rol(Delta[i & 3], i); + this._rk[i, 0] = t[0] = Rol(t[0] + temp, 1); + this._rk[i, 1] = this._rk[i, 3] = this._rk[i, 5] = t[1] = Rol(t[1] + Rol(temp, 1), 3); + this._rk[i, 2] = t[2] = Rol(t[2] + Rol(temp, 2), 6); + this._rk[i, 4] = t[3] = Rol(t[3] + Rol(temp, 3), 11); + } + break; + case 24: + for (var i = 0; i < 28; ++i) { + var temp = Rol(Delta[i % 6], i); + this._rk[i, 0] = t[0] = Rol(t[0] + temp, 1); + this._rk[i, 1] = t[1] = Rol(t[1] + Rol(temp, 1), 3); + this._rk[i, 2] = t[2] = Rol(t[2] + Rol(temp, 2), 6); + this._rk[i, 3] = t[3] = Rol(t[3] + Rol(temp, 3), 11); + this._rk[i, 4] = t[4] = Rol(t[4] + Rol(temp, 4), 13); + this._rk[i, 5] = t[5] = Rol(t[5] + Rol(temp, 5), 17); + } + break; + default: + for (var i = 0; i < 32; ++i) { + var temp = Rol(Delta[i & 7], i & 0x1f); + this._rk[i, 0] = t[(6 * i + 0) & 7] = Rol(t[(6 * i + 0) & 7] + temp, 1); + this._rk[i, 1] = t[(6 * i + 1) & 7] = Rol(t[(6 * i + 1) & 7] + Rol(temp, 1), 3); + this._rk[i, 2] = t[(6 * i + 2) & 7] = Rol(t[(6 * i + 2) & 7] + Rol(temp, 2), 6); + this._rk[i, 3] = t[(6 * i + 3) & 7] = Rol(t[(6 * i + 3) & 7] + Rol(temp, 3), 11); + this._rk[i, 4] = t[(6 * i + 4) & 7] = Rol(t[(6 * i + 4) & 7] + Rol(temp, 4), 13); + this._rk[i, 5] = t[(6 * i + 5) & 7] = Rol(t[(6 * i + 5) & 7] + Rol(temp, 5), 17); + } + break; + } + } + + public void EncryptBlock(ReadOnlySpan input, Span output) { + var b0 = BinaryPrimitives.ReadUInt32LittleEndian(input); + var b1 = BinaryPrimitives.ReadUInt32LittleEndian(input[4..]); + var b2 = BinaryPrimitives.ReadUInt32LittleEndian(input[8..]); + var b3 = BinaryPrimitives.ReadUInt32LittleEndian(input[12..]); + for (var i = 0; i < this._rounds; i += 4) { + b3 = Ror((b2 ^ this._rk[i, 4]) + (b3 ^ this._rk[i, 5]), 3); + b2 = Ror((b1 ^ this._rk[i, 2]) + (b2 ^ this._rk[i, 3]), 5); + b1 = Rol((b0 ^ this._rk[i, 0]) + (b1 ^ this._rk[i, 1]), 9); + b0 = Ror((b3 ^ this._rk[i + 1, 4]) + (b0 ^ this._rk[i + 1, 5]), 3); + b3 = Ror((b2 ^ this._rk[i + 1, 2]) + (b3 ^ this._rk[i + 1, 3]), 5); + b2 = Rol((b1 ^ this._rk[i + 1, 0]) + (b2 ^ this._rk[i + 1, 1]), 9); + b1 = Ror((b0 ^ this._rk[i + 2, 4]) + (b1 ^ this._rk[i + 2, 5]), 3); + b0 = Ror((b3 ^ this._rk[i + 2, 2]) + (b0 ^ this._rk[i + 2, 3]), 5); + b3 = Rol((b2 ^ this._rk[i + 2, 0]) + (b3 ^ this._rk[i + 2, 1]), 9); + b2 = Ror((b1 ^ this._rk[i + 3, 4]) + (b2 ^ this._rk[i + 3, 5]), 3); + b1 = Ror((b0 ^ this._rk[i + 3, 2]) + (b1 ^ this._rk[i + 3, 3]), 5); + b0 = Rol((b3 ^ this._rk[i + 3, 0]) + (b0 ^ this._rk[i + 3, 1]), 9); + } + BinaryPrimitives.WriteUInt32LittleEndian(output, b0); + BinaryPrimitives.WriteUInt32LittleEndian(output[4..], b1); + BinaryPrimitives.WriteUInt32LittleEndian(output[8..], b2); + BinaryPrimitives.WriteUInt32LittleEndian(output[12..], b3); + } +} diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Egg/EggEntry.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Egg/EggEntry.cs index 1dc1a4da8..7a64af6a0 100644 --- a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Egg/EggEntry.cs +++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Egg/EggEntry.cs @@ -54,6 +54,9 @@ public sealed class EggEntry { _ => $"Method{PrimaryAlgorithm}", }; + /// The entry's encryption field, when it has one. + internal EggEncryption? Encryption { get; set; } + /// The physical data blocks that make up the entry, in order. internal List Blocks { get; } = []; } diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Egg/EggFormatDescriptor.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Egg/EggFormatDescriptor.cs index e766308e9..f63c4364c 100644 --- a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Egg/EggFormatDescriptor.cs +++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Egg/EggFormatDescriptor.cs @@ -44,7 +44,7 @@ public sealed class EggFormatDescriptor : public FormatCapabilities Capabilities => FormatCapabilities.CanList | FormatCapabilities.CanExtract | FormatCapabilities.CanCreate | FormatCapabilities.CanTest | FormatCapabilities.SupportsMultipleEntries | - FormatCapabilities.SupportsDirectories; + FormatCapabilities.SupportsDirectories | FormatCapabilities.SupportsPassword; /// /// Gets the default extension. /// @@ -79,7 +79,7 @@ public sealed class EggFormatDescriptor : /// /// Gets the description. /// - public string Description => "ESTsoft ALZip EGG archive (Store/Deflate read + write)"; + public string Description => "ESTsoft ALZip EGG archive (reads stored/deflate/bzip2/AZO/LZMA, ZIP 2.0/AES/LEA encryption, solid; writes stored/deflate)"; /// /// Lists the entries in the supplied container. @@ -102,7 +102,7 @@ public void Extract(Stream stream, string outputDir, string? password, string[]? CreateSafeDirectory(outputDir, e.Name); continue; } - WriteFile(outputDir, e.Name, r.Extract(e)); + WriteFile(outputDir, e.Name, r.Extract(e, password)); } } @@ -118,7 +118,7 @@ public Stream OpenEntry(Stream archive, string entryName, string? password) { foreach (var e in r.Entries) { if (e.IsDirectory) continue; if (!string.Equals(e.Name, entryName, StringComparison.OrdinalIgnoreCase)) continue; - var bytes = r.Extract(e); + var bytes = r.Extract(e, password); return new Compression.Registry.Streaming.BoundedEntryStream( new MemoryStream(bytes, writable: false), bytes.Length, leaveOpen: false); } diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Egg/EggReader.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Egg/EggReader.cs index 72dd259a3..300babb05 100644 --- a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Egg/EggReader.cs +++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Egg/EggReader.cs @@ -93,7 +93,14 @@ public EggReader(Stream stream, bool leaveOpen = false) { public bool IsSplit { get; private set; } /// True when a global (archive-level) encryption header is present. - public bool IsGloballyEncrypted { get; private set; } + public bool IsGloballyEncrypted => this.GlobalEncryption != null; + + /// True when the members are compressed as one solid stream. + public bool IsSolid { get; private set; } + + internal EggEncryption? GlobalEncryption { get; private set; } + + private byte[]? _solidData; private void _ReadArchive() { _stream.Position = 0; @@ -107,11 +114,11 @@ private void _ReadArchive() { _ = _reader.ReadUInt32(); // reserved // EGG-header extra fields (split / solid / global encryption) up to the end-marker. - _SkipExtraFields((field, _, _, _) => { + _SkipExtraFields((field, _, size, dataStart) => { switch (field) { case SplitMagic: this.IsSplit = true; break; - case SolidMagic: break; - case EncryptMagic: this.IsGloballyEncrypted = true; break; + case SolidMagic: this.IsSolid = true; break; + case EncryptMagic: this.GlobalEncryption = EggEncryption.Parse(_ReadField(size, dataStart)); break; } }); @@ -120,6 +127,10 @@ private void _ReadArchive() { var token = _reader.ReadUInt32(); if (token == EndMarker) break; // end of archive + if (token == CommentMagic) { + this._SkipOneField(); // archive comment record + continue; + } if (token != FileHeaderMagic) throw new InvalidDataException($"Unexpected token 0x{token:X8} where a file header was expected."); @@ -139,7 +150,10 @@ private void _ReadArchive() { case FilenameMagic: _ParseFilename(bitFlag, size, dataStart, entry); break; case WindowsInfoMagic: _ParseWindowsInfo(entry); break; case PosixInfoMagic: _ParsePosixInfo(entry); break; - case EncryptMagic: entry.IsEncrypted = true; break; + case EncryptMagic: + entry.IsEncrypted = true; + entry.Encryption = EggEncryption.Parse(_ReadField(size, dataStart)); + break; } }); @@ -157,6 +171,10 @@ private void _ReadArchive() { private void _ReadBlocks(EggEntry entry) { while (_stream.Position + 4 <= _stream.Length) { var token = _reader.ReadUInt32(); + if (token == CommentMagic) { + this._SkipOneField(); // a comment record between the file header and its blocks + continue; + } if (token != BlockHeaderMagic) { _stream.Position -= 4; // belongs to the next file header or the archive end-marker break; @@ -185,57 +203,128 @@ private void _ReadBlocks(EggEntry entry) { } /// - /// Extracts and decompresses an entry into a byte array. Store and Deflate blocks - /// are decoded natively; any other algorithm, or an encrypted / split-volume entry, - /// raises rather than returning wrong bytes. - /// Every decoded block is checked against its declared size and CRC-32. + /// Extracts and decompresses an entry: stored, deflate, bzip2, AZO and LZMA blocks, ZIP 2.0 / AES / + /// LEA encryption, solid archives. Split (multi-volume) members raise + /// rather than returning wrong bytes. Every decoded block is + /// checked against its declared size and CRC-32. /// - public byte[] Extract(EggEntry entry) { + public byte[] Extract(EggEntry entry, string? password = null) { ArgumentNullException.ThrowIfNull(entry); - if (entry.IsDirectory) + if (entry.IsDirectory || entry.UncompressedSize == 0 && entry.Blocks.Count == 0 && !this.IsSolid) return []; if (this.IsSplit) throw new NotSupportedException("EGG split (multi-volume) archives are not supported for extraction."); - if (entry.IsEncrypted) - throw new NotSupportedException($"EGG entry '{entry.Name}' is encrypted; extraction is not supported."); + if (this.IsSolid) { + this._solidData ??= this._DecodeSolid(password); + var offset = 0L; + foreach (var e in this._entries) { + if (ReferenceEquals(e, entry)) + return this._solidData.AsSpan((int)offset, (int)e.UncompressedSize).ToArray(); + if (!e.IsDirectory) + offset += e.UncompressedSize; + } + throw new ArgumentException("Entry does not belong to this archive.", nameof(entry)); + } + + var encryption = entry.Encryption ?? this.GlobalEncryption; using var output = new MemoryStream(); - var crc = new Crc32(); - foreach (var block in entry.Blocks) { - if (block.CompressedSize > int.MaxValue) - throw new InvalidDataException($"EGG entry '{entry.Name}' has a block too large for in-memory extraction."); - _stream.Position = block.DataOffset; - var compressed = new byte[(int)block.CompressedSize]; - _ReadExactly(compressed); - - var plain = block.Algorithm switch { - 0 => compressed, // Store - 1 => DeflateDecompressor.Decompress(compressed), // Deflate (raw) - _ => throw new NotSupportedException( - $"EGG entry '{entry.Name}' uses unsupported compression '{new EggEntry { PrimaryAlgorithm = block.Algorithm }.MethodName}'."), - }; + foreach (var block in entry.Blocks) + output.Write(this._DecodeBlock(entry.Name, block, encryption, password)); - if (plain.LongLength != block.UncompressedSize) - throw new InvalidDataException( - $"EGG entry '{entry.Name}' block expands to {plain.LongLength} bytes; header declares {block.UncompressedSize}."); - crc.Reset(); - crc.Update(plain); - if (crc.Value != block.Crc32) - throw new InvalidDataException( - $"EGG entry '{entry.Name}' failed CRC-32 validation (stored 0x{block.Crc32:X8}, computed 0x{crc.Value:X8})."); + if (output.Length != entry.UncompressedSize) + throw new InvalidDataException( + $"EGG entry '{entry.Name}' expands to {output.Length} bytes; file header declares {entry.UncompressedSize}."); + return output.ToArray(); + } + + /// Decodes the one compressed stream of a solid archive (the blocks after the last file header). + private byte[] _DecodeSolid(string? password) { + var total = this._entries.Where(e => !e.IsDirectory).Sum(e => e.UncompressedSize); + var holder = this._entries.LastOrDefault(e => e.Blocks.Count > 0) + ?? throw new InvalidDataException("EGG solid archive has no data block."); + var encryption = this._entries.Select(e => e.Encryption).FirstOrDefault(e => e != null) ?? this.GlobalEncryption; + using var output = new MemoryStream(); + foreach (var block in holder.Blocks) + output.Write(this._DecodeBlock("solid stream", block, encryption, password)); + if (output.Length != total) + throw new InvalidDataException($"EGG solid stream expands to {output.Length} bytes; the file headers declare {total}."); + return output.ToArray(); + } - output.Write(plain, 0, plain.Length); + private byte[] _DecodeBlock(string name, EggBlock block, EggEncryption? encryption, string? password) { + if (block.CompressedSize > int.MaxValue || block.UncompressedSize > int.MaxValue) + throw new NotSupportedException($"EGG entry '{name}' has a block too large for in-memory extraction."); + _stream.Position = block.DataOffset; + var packed = new byte[(int)block.CompressedSize]; + _ReadExactly(packed); + + if (encryption != null) { + if (string.IsNullOrEmpty(password)) + throw new InvalidOperationException($"EGG entry '{name}' is encrypted ({encryption.MethodName}); a password is required."); + packed = encryption.Decrypt(packed, password); } - if (output.Length != entry.UncompressedSize) + byte[] plain; + try { + plain = block.Algorithm switch { + 0 => packed, + 1 => DeflateDecompressor.Decompress(packed), + 2 => _Bzip2(packed), + 3 => Compression.Core.Dictionary.Azo.AzoDecoder.Decompress(packed, block.UncompressedSize), + 4 => _Lzma(packed, block.UncompressedSize), + _ => throw new NotSupportedException($"EGG entry '{name}' uses unknown compression method {block.Algorithm}."), + }; + } catch (Exception e) when (e is not (NotSupportedException or InvalidDataException or InvalidOperationException)) { + throw new InvalidDataException($"EGG entry '{name}': corrupt compressed data.", e); + } + + if (plain.LongLength != block.UncompressedSize) throw new InvalidDataException( - $"EGG entry '{entry.Name}' expands to {output.Length} bytes; file header declares {entry.UncompressedSize}."); + $"EGG entry '{name}' block expands to {plain.LongLength} bytes; header declares {block.UncompressedSize}."); + if (Crc32.Compute(plain) != block.Crc32) + throw new InvalidDataException( + $"EGG entry '{name}' failed CRC-32 validation{(encryption != null ? " (wrong password?)" : "")}."); + return plain; + } + + private static byte[] _Bzip2(byte[] packed) { + using var input = new MemoryStream(packed, writable: false); + using var bz = new FileFormat.Bzip2.Bzip2Stream(input, Compression.Core.Streams.CompressionStreamMode.Decompress); + using var output = new MemoryStream(); + bz.CopyTo(output); return output.ToArray(); } + /// EGG's LZMA block: 4 bytes (version and property size), the 5 LZMA properties, then raw LZMA. + private static byte[] _Lzma(byte[] packed, long size) { + if (packed.Length < 9) + throw new InvalidDataException("Truncated EGG LZMA block."); + using var input = new MemoryStream(packed, 9, packed.Length - 9, writable: false); + return new Compression.Core.Dictionary.Lzma.LzmaDecoder(input, packed[4..9], size).Decode(); + } + + private byte[] _ReadField(long size, long dataStart) { + if (size > 1 << 20) + throw new InvalidDataException("EGG extra field too large."); + _stream.Position = dataStart; + var data = new byte[size]; + _ReadExactly(data); + return data; + } + // ── Extra-field iteration ──────────────────────────────────────────────── + /// Skips one record whose magic was just read: flag byte, 2- or 4-byte size, data. + private void _SkipOneField() { + var bitFlag = _reader.ReadByte(); + long size = (bitFlag & 0x01) != 0 ? _reader.ReadUInt32() : _reader.ReadUInt16(); + if (_stream.Position > _stream.Length - size) + throw new InvalidDataException("EGG record extends beyond the input stream."); + _stream.Position += size; + } + private void _SkipExtraFields(Action? onField) { while (true) { if (_stream.Position + 4 > _stream.Length) @@ -252,12 +341,15 @@ private void _SkipExtraFields(Action? onField) { throw new InvalidDataException("EGG archive ends inside an extra-field size."); var size = sizeWidth == 4 ? _reader.ReadUInt32() : _reader.ReadUInt16(); var dataStart = _stream.Position; - if (dataStart > _stream.Length - size) + // A filename field's size counts the name only; the code page (flag 0x08) and the parent-path + // id (flag 0x10) precede it uncounted, as ALZip writes them (and EggDotNet reads them). + var uncounted = field == FilenameMagic ? ((bitFlag & 0x08) != 0 ? 2 : 0) + ((bitFlag & 0x10) != 0 ? 4 : 0) : 0; + if (dataStart > _stream.Length - size - uncounted) throw new InvalidDataException($"EGG extra field 0x{field:X8} extends beyond the input stream."); onField?.Invoke(field, bitFlag, size, dataStart); - _stream.Position = dataStart + size; // resync to the field end regardless of handling + _stream.Position = dataStart + size + uncounted; // resync to the field end regardless of handling } } @@ -281,7 +373,7 @@ private void _ParseFilename(byte bitFlag, long size, long dataStart, EggEntry en if (parentLen == 4) _ = _reader.ReadUInt32(); // parent-path id (relative-path linkage; not resolved here) - var nameLen = size - localeLen - parentLen; + var nameLen = size; if (nameLen < 0 || nameLen > int.MaxValue) throw new InvalidDataException("EGG filename field has an invalid length."); var nameBytes = _reader.ReadBytes((int)nameLen); diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Egg/EggWriter.cs b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Egg/EggWriter.cs index eaab0d30e..3104858be 100644 --- a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Egg/EggWriter.cs +++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Egg/EggWriter.cs @@ -103,7 +103,9 @@ public void Finish() { WriteWindowsInfo(writer, entry.LastModifiedUtc, entry.IsDirectory); writer.Write(EggReader.EndMarker); // end of file sub-headers - if (!entry.IsDirectory) + // ALZip writes no block for a directory or an empty file, and readers (EggDotNet among them) + // only look for blocks when the file header announces a non-zero size. + if (!entry.IsDirectory && entry.Data.Length > 0) WriteBlock(writer, entry.Data, entry.Method, entry.Level); } diff --git a/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Egg/THIRD-PARTY-NOTICE.EggDotNet.txt b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Egg/THIRD-PARTY-NOTICE.EggDotNet.txt new file mode 100644 index 000000000..5970171eb --- /dev/null +++ b/Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Egg/THIRD-PARTY-NOTICE.EggDotNet.txt @@ -0,0 +1,29 @@ +The AZO decoder (Compression.Core/Dictionary/Azo/AzoDecoder.cs) and the EGG encryption code +(Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Egg/EggCrypto.cs: ZIP 2.0, AES and LEA variants and +the LEA block cipher) are converted to C# from EggDotNet; the EGG test archives under +Compression.Tests/Egg/ReferenceVectors come from its test corpus: + + Upstream: https://github.com/akolman/EggDotNet (master, fetched 2026-10-23) + Licence: MIT + +MIT License + +Copyright (c) 2026 Andrew Kolman (akolman) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Hawkynt.FileFormats.Archives/REFERENCE.md b/Hawkynt.FileFormats.Archives/REFERENCE.md index 464eb0ba6..799f1769a 100644 --- a/Hawkynt.FileFormats.Archives/REFERENCE.md +++ b/Hawkynt.FileFormats.Archives/REFERENCE.md @@ -14493,9 +14493,10 @@ Implements `IDisposable`. | `EggReader` | `EggReader(Stream stream, bool leaveOpen = false)` | Opens an EGG archive over the given seekable stream and parses its directory. | | `Entries` | `IReadOnlyList Entries { get; }` | File entries discovered in the archive. | | `IsGloballyEncrypted` | `bool IsGloballyEncrypted { get; }` | True when a global (archive-level) encryption header is present. | +| `IsSolid` | `bool IsSolid { get; }` | True when the members are compressed as one solid stream. | | `IsSplit` | `bool IsSplit { get; }` | True when the archive is a member of a split (multi-volume) set. | | `Dispose` | `void Dispose()` | | -| `Extract` | `byte[] Extract(EggEntry entry)` | Extracts and decompresses an entry into a byte array. Store and Deflate blocks are decoded natively; any other algorithm, or an encrypted / split-volume entry, raises `NotSupportedException` rather than returning wrong bytes. Every decoded block is checked against its declared size and CRC-32. | +| `Extract` | `byte[] Extract(EggEntry entry, string password = null)` | Extracts and decompresses an entry: stored, deflate, bzip2, AZO and LZMA blocks, ZIP 2.0 / AES / LEA encryption, solid archives. Split (multi-volume) members raise `NotSupportedException` rather than returning wrong bytes. Every decoded block is checked against its declared size and CRC-32. | #### `EggWriter` diff --git a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs index 6c7d4b452..1c9e17266 100644 --- a/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs +++ b/Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.FirmwareHex/FirmwareHexWriter.cs @@ -33,6 +33,7 @@ public static class FirmwareHexWriter { /// one the reader rendered. An input list with no payload describes an image /// with no data, which both formats can write. /// + /// More than one non-metadata payload is supplied. public static FirmwareImage ImageFrom(IReadOnlyList inputs, string sourceFormat) { ArgumentNullException.ThrowIfNull(inputs); @@ -53,7 +54,9 @@ public static FirmwareImage ImageFrom(IReadOnlyList inputs, st } // Any other single input is the payload, whatever it is called: a caller // converting a .bin into a .hex should not have to rename it first. - payload ??= input.ReadContent(); + if (payload is not null) + throw new InvalidDataException($"{sourceFormat} represents one logical firmware payload; multiple payload inputs are ambiguous."); + payload = input.ReadContent(); } List<(uint Address, byte[] Data)> segments; diff --git a/docs/LICENCE-PROVENANCE.md b/docs/LICENCE-PROVENANCE.md index ca50e4c15..8435dcf83 100644 --- a/docs/LICENCE-PROVENANCE.md +++ b/docs/LICENCE-PROVENANCE.md @@ -31,7 +31,7 @@ package carries; everything else keeps a `THIRD-PARTY-NOTICE..txt` beside | Class | Upstream rows | Notes | |---|---|---| -| A | 18 | | +| A | 19 | | B | 17 | FFmpeg alone covers 38 codec folders | | C | 3 | specifications: `docs/CVF-CODECS-ON-DISK.md` (rewrite already under way), `docs/IT-SAMPLE-COMPRESSION-SPEC.md`, `docs/FDS-EXPANSION-SOUND-SPEC.md` | | R | 8 | | @@ -70,6 +70,7 @@ package carries; everything else keeps a `THIRD-PARTY-NOTICE..txt` beside | Claim site | Upstream | Licence | Where the notice is | |---|---|---|---| +| `Compression.Core/Dictionary/Azo/AzoDecoder.cs`, `Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Egg/EggCrypto.cs` | EggDotNet (Andrew Kolman), AZO decoding and ZIP/AES/LEA encryption, fetched 2026-10-23 as recorded in the notices | MIT | `Compression.Core/Dictionary/Azo/THIRD-PARTY-NOTICE.EggDotNet.txt`; `Hawkynt.FileFormats.Archives/FileFormats/FileFormat.Egg/THIRD-PARTY-NOTICE.EggDotNet.txt` | | `Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidBootImage/AndroidBootImage.cs`, `Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidBootImage/AndroidBootImageMetadata.cs` | AOSP mkbootimg `bootimg.h`, `mkbootimg.py` and `unpack_bootimg.py`, revision `d2bb0af5ba6d3198a3e99529c97eda1be0b5a093` | BSD-3-Clause and Apache-2.0 | `Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidBootImage/THIRD-PARTY-NOTICE.mkbootimg.txt` | | `Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidVbmeta/AvbVbmeta.cs`, `Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidVbmeta/AvbDescriptorText.cs` | AOSP Android Verified Boot headers and `avbtool.py`, revision `c5066a96caa7bf4150c0a8cc8cc14ab81733fdc7` | MIT | `Hawkynt.FileFormats.FileSystems/FileFormats/FileFormat.AndroidVbmeta/THIRD-PARTY-NOTICE.avb.txt` | | `Compression.Core/Dictionary/Lzbitmap/` | libzbitmap (Ernesto A. Fernández, Corellium) | MIT | `THIRD-PARTY-NOTICE.libzbitmap.txt` | diff --git a/docs/catalogue/reference-formats.csv b/docs/catalogue/reference-formats.csv index d168a2e19..2d2f31faa 100644 --- a/docs/catalogue/reference-formats.csv +++ b/docs/catalogue/reference-formats.csv @@ -42,7 +42,7 @@ zoo,zoo,archive,,,R,,R,x-fmt/269,Y,4,https://en.wikipedia.org/wiki/Zoo_(file_for LBR (CP/M library),lbr;lqr;lzr,archive,,,R,,R,,Y,3,https://en.wikipedia.org/wiki/LBR_(file_format),Lbr,Y,Y,Y,OK,, LBR (Commodore library),lbr,archive,,,,,,,Y,1,,,,,,MISSING,, ALZip,alz,archive,,,R,,R,,RW,3,https://en.wikipedia.org/wiki/ALZip,AlZip,Y,Y,Y,PARTIAL,,store and deflate only (AlZipReader.cs) -EGG (ALZip),egg,archive,,,R,,,,,1,https://en.wikipedia.org/wiki/ALZip,Egg,Y,Y,,PARTIAL,,"store and deflate only; bzip2/LZMA/AZO, encryption and split archives unsupported (EggReader.cs)" +EGG (ALZip),egg,archive,,,R,,,,,1,https://en.wikipedia.org/wiki/ALZip,Egg,Y,Y,,PARTIAL,,"reads store/deflate/bzip2/AZO/LZMA, solid, ZIP 2.0/AES/LEA encryption (EggDotNet corpus); writer store/deflate only (EggDotNet reads it back); split (multi-volume) archives unsupported (EggReader.cs)" StuffIt (classic + 5),sit;sea,archive,,,R,,R,fmt/1459;fmt/1460;fmt/639,RW,4,https://en.wikipedia.org/wiki/StuffIt,StuffIt,Y,Y,Y,PARTIAL,,store and RLE only; LZW/Huffman/LZAH/Arsenic/DES methods throw (StuffItReader.cs) StuffIt X,sitx,archive,,,R,,R,fmt/399,RW,4,https://en.wikipedia.org/wiki/StuffIt,StuffItX,Y,Y,,PARTIAL,,stored and deflate only; Brimstone/Cyanide/Darkhorse/Iron throw (StuffItXReader.cs) DiskDoubler,dd,archive,,,R,,,,Y,2,https://en.wikipedia.org/wiki/DiskDoubler,DiskDoubler,Y,Y,,PARTIAL,,"single files only (no DDAR/DDA2 archives); methods 0, 1, 6, 9 decoded and checked against unar on real files; 2-5, 7, 8, 10 and delta refused (DiskDoublerReader.cs)"