+ PEA (PeaZip) archives: read and create

Compression.Tests/Compression.Tests.csproj

@@ -150,6 +150,10 @@
     <EmbeddedResource Include="Zfs\ReferenceVectors\*.gz;Zfs\ReferenceVectors\*.manifest">
       <LogicalName>ZfsVectors.%(Filename)%(Extension)</LogicalName>
     </EmbeddedResource>
+    <!-- PEA archives written by PeaZip 11.4; see Pea/ReferenceVectors/README.md. -->
+    <EmbeddedResource Include="Pea\ReferenceVectors\*.pea">
+      <LogicalName>PeaVectors.%(Filename)%(Extension)</LogicalName>
+    </EmbeddedResource>
     <!-- 7z archives 7-Zip 26.03, 7-Zip-zstd 26.02 and py7zr wrote; see SevenZip/ReferenceVectors/README.md. -->
     <EmbeddedResource Include="SevenZip\ReferenceVectors\*.7z;SevenZip\ReferenceVectors\manifest.txt">
       <LogicalName>SevenZipVectors.%(Filename)%(Extension)</LogicalName>

Compression.Tests/Hashing/WhirlpoolTests.cs

@@ -0,0 +1,23 @@
+using Hawkynt.Algorithms.Hashing;
+
+namespace Compression.Tests.Hashing;
+
+/// <summary>
+/// Whirlpool against the ISO/NESSIE vectors (cross-checked with OpenSSL's legacy provider), at the
+/// tail lengths around its padding boundary: 0x80 plus a 256-bit length needs a second final block
+/// once 32 or more message bytes are left over.
+/// </summary>
+[TestFixture]
+public class WhirlpoolTests {
+  [TestCase("", "19fa61d75522a4669b44e39c1d2e1726c530232130d407f89afee0964997f7a73e83be698b288febcf88e3e03c4f0757ea8964e59b63d93708b138cc42a66eb3")]
+  [TestCase("abc", "4e2448a4c6f486bb16b6562c73b4020bf3043e3a731bce721ae1b303d97e6d4c7181eebdb6c57e277d0e34957114cbd6c797fc9d95d8b582d225292076d4eef5")]
+  [TestCase("The quick brown fox jumps over the lazy dog", "b97de512e91e3828b40d2b0fdce9ceb3c4a71f9bea8d88e75c4fa854df36725fd2b52eb6544edcacd6f8beddfea403cb55ae31f03ad62a5ef54e42ee82c3fb35")]
+  [Category("HappyPath")]
+  public void GivenReferenceInput_WhenHashed_ThenMatchesTheReferenceDigest(string input, string expected)
+    => Assert.That(Convert.ToHexStringLower(Whirlpool.Compute(System.Text.Encoding.ASCII.GetBytes(input))), Is.EqualTo(expected));
+
+  [Test, Category("Boundary")]
+  public void GivenOneBlockPlus41Bytes_WhenHashed_ThenTheTailNeedsTwoPaddingBlocks()
+    => Assert.That(Convert.ToHexStringLower(Whirlpool.Compute(Enumerable.Repeat((byte)'a', 105).ToArray())),
+      Is.EqualTo("d352ca65c7d77ae0c27a8e1ce3b8e887f05bef753e28c7b7c7ff112f1ff4c1bcaf83f749ad1e6e397d210ea14da44c4c5615a77831ad80a132e3a67cef2b1878"));
+}

Compression.Tests/Pea/PeaTests.cs

@@ -0,0 +1,139 @@
+using System.Reflection;
+using System.Security.Cryptography;
+using Compression.Registry;
+using FileFormat.Pea;
+
+namespace Compression.Tests.Pea;
+
+/// <summary>
+/// PEA: archives PeaZip 11.4 wrote with every compression level and most check algorithms
+/// (Pea/ReferenceVectors), our writer, and PeaZip's own pea (headless) as the oracle for what we
+/// write.
+/// </summary>
+[TestFixture]
+public class PeaTests {
+
+  private static readonly byte[] Fox = System.Text.Encoding.ASCII.GetBytes(string.Concat(Enumerable.Repeat("The quick brown fox jumps over the lazy dog.\n", 300)));
+  private static readonly byte[] Hello = "hello pea\n"u8.ToArray();
+  private static readonly byte[] Nested = "nested\n"u8.ToArray();
+  private static readonly DateTime SourceTime = new(2001, 2, 3, 4, 5, 6, DateTimeKind.Utc);
+
+  private string _tmpDir = null!;
+
+  [SetUp]
+  public void Setup() {
+    this._tmpDir = Path.Combine(Path.GetTempPath(), $"cwb_pea_{Guid.NewGuid():N}");
+    Directory.CreateDirectory(this._tmpDir);
+  }
+
+  [TearDown]
+  public void Teardown() {
+    try { Directory.Delete(this._tmpDir, true); } catch { /* best effort */ }
+  }
+
+  [TestCase("store.pea")]    // PCOMPRESS0, CRC32 volume/object, no stream check
+  [TestCase("deflate3.pea")] // PCOMPRESS1, Adler-32 volume, CRC-64 objects, SHA-256 stream
+  [TestCase("deflate6.pea")] // PCOMPRESS2, CRC32, CRC32, SHA-256
+  [TestCase("deflate9.pea")] // PCOMPRESS3, SHA-1 volume, MD5 objects, SHA-512 stream
+  [TestCase("sha3.pea")]     // SHA3-256 volume, SHA3-512 objects, BLAKE2b stream
+  [TestCase("misc.pea")]     // RIPEMD-160 volume, Whirlpool objects, BLAKE2s stream
+  [Category("ThemVsUs")]
+  public void GivenArchiveWrittenByPeaZip_WhenRead_ThenEveryObjectAndCheckVerifies(string vector) {
+    var entries = PeaArchive.Read(new MemoryStream(LoadVector(vector)));
+
+    Assert.That(entries.Select(e => (e.Path, e.IsDirectory)), Is.EquivalentTo(new[] {
+      ("hello.txt", false), ("fox.txt", false), ("empty.txt", false), ("sub", true), ("sub/n.txt", false) }));
+    Assert.That(entries.Single(e => e.Path == "hello.txt").Data, Is.EqualTo(Hello));
+    Assert.That(entries.Single(e => e.Path == "fox.txt").Data, Is.EqualTo(Fox));
+    Assert.That(entries.Single(e => e.Path == "empty.txt").Data, Is.Empty);
+    Assert.That(entries.Single(e => e.Path == "sub/n.txt").Data, Is.EqualTo(Nested));
+    Assert.That(entries.Single(e => e.Path == "fox.txt").LastModified, Is.EqualTo(SourceTime));
+  }
+
+  [Test, Category("ThemVsUs")]
+  public void GivenMultiBlockArchiveWrittenByPeaZip_WhenRead_ThenAll3MiBDecode() {
+    var big = PeaArchive.Read(new MemoryStream(LoadVector("big.pea"))).Single();
+    var expected = Enumerable.Range(0, 3_000_000).Select(i => (byte)((i * 131 + (i >> 7)) % 256)).ToArray();
+    Assert.That(big.Path, Is.EqualTo("big.bin"));
+    Assert.That(big.Data, Is.EqualTo(expected));
+  }
+
+  [Test, Category("Exception")]
+  public void GivenFlippedDataByte_WhenRead_ThenAChecksumRefusesIt() {
+    var bytes = LoadVector("store.pea");
+    var at = bytes.AsSpan().IndexOf("The quick"u8);
+    bytes[at + 5] ^= 0x01;
+    Assert.That(() => PeaArchive.Read(new MemoryStream(bytes)), Throws.InstanceOf<InvalidDataException>());
+  }
+
+  [Test, Category("Exception")]
+  public void GivenEncryptedStreamHeader_WhenRead_ThenRefusedAsUnsupported() {
+    var bytes = LoadVector("deflate6.pea");
+    bytes[18] = 0x31; // EAX
+    Assert.That(() => PeaArchive.Read(new MemoryStream(bytes)), Throws.InstanceOf<NotSupportedException>());
+  }
+
+  [Test, Category("Exception")]
+  public void GivenNotPea_WhenRead_ThenRefused()
+    => Assert.That(() => PeaArchive.Read(new MemoryStream(new byte[64])), Throws.InstanceOf<InvalidDataException>());
+
+  // ── our writer ──────────────────────────────────────────────────────
+
+  private static List<PeaArchive.Input> Inputs() => [
+    new("hello.txt", Hello, SourceTime),
+    new("dir/fox.txt", Fox, SourceTime),
+    new("dir/deep/empty.bin", [], SourceTime),
+    new("big.bin", Enumerable.Range(0, 2_500_000).Select(i => (byte)(i * 7 + (i >> 9))).ToArray(), SourceTime),
+  ];
+
+  [TestCase(0, PeaArchive.Check.Crc32, PeaArchive.Check.None, PeaArchive.Check.Crc32)]
+  [TestCase(1, PeaArchive.Check.Crc64, PeaArchive.Check.Sha256, PeaArchive.Check.Adler32)]
+  [TestCase(2, PeaArchive.Check.Crc32, PeaArchive.Check.Sha256, PeaArchive.Check.Crc32)]
+  [TestCase(3, PeaArchive.Check.Md5, PeaArchive.Check.Sha512, PeaArchive.Check.Sha1)]
+  [Category("RoundTrip")]
+  public void GivenFiles_WhenWrittenAndRead_ThenEveryByteRoundTrips(int compression, PeaArchive.Check obj, PeaArchive.Check stream, PeaArchive.Check volume) {
+    using var ms = new MemoryStream();
+    PeaArchive.Write(ms, Inputs(), compression, obj, stream, volume);
+    var entries = PeaArchive.Read(new MemoryStream(ms.ToArray()));
+    foreach (var input in Inputs())
+      Assert.That(entries.Single(e => e.Path == input.Path).Data, Is.EqualTo(input.Data), input.Path);
+    Assert.That(entries.Where(e => e.IsDirectory).Select(e => e.Path), Is.EquivalentTo(new[] { "dir", "dir/deep" }));
+  }
+
+  [Test, Category("Exception")]
+  public void GivenPassword_WhenCreated_ThenRefused()
+    => Assert.That(() => new PeaFormatDescriptor().Create(new MemoryStream(), [ArchiveInputInfo.InMemory("a", [1])], new FormatCreateOptions { Password = "x" }),
+      Throws.InstanceOf<NotSupportedException>());
+
+  [Test, Category("ExternalInterop")]
+  public void GivenOurArchive_WhenPeaZipExtractsIt_ThenEveryFileMatches() {
+    var pea = RequirePea();
+    var path = Path.Combine(this._tmpDir, "ours.pea");
+    using (var f = File.Create(path))
+      PeaArchive.Write(f, Inputs());
+    var outDir = Path.Combine(this._tmpDir, "out");
+    Directory.CreateDirectory(outDir);
+
+    var r = FsInteropToolbox.RunWsl($"timeout 120 xvfb-run -a '{pea}' UNPEA '{FsInteropToolbox.WinToWsl(path)}' '{FsInteropToolbox.WinToWsl(outDir)}/' RESETDATE RESETATTR EXTRACT2DIR HIDDEN '' ''");
+    Assert.That(r.ExitCode, Is.EqualTo(0), $"pea UNPEA failed:\n{r.StdOut}\n{r.StdErr}");
+    var root = Path.Combine(outDir, "ours.pea");
+    foreach (var input in Inputs())
+      Assert.That(File.ReadAllBytes(Path.Combine(root, input.Path.Replace('/', Path.DirectorySeparatorChar))), Is.EqualTo(input.Data), input.Path);
+  }
+
+  private static string RequirePea() {
+    if (OperatingSystem.IsWindows() && !FsInteropToolbox.WslAvailable) Assert.Ignore("WSL is not available.");
+    var pea = Environment.GetEnvironmentVariable("PEAZIP_PEA") ?? "/opt/cwb_peazip/peazip_portable-11.4.0.LINUX.GTK2.x86_64/pea";
+    var probe = FsInteropToolbox.RunWsl($"test -x '{pea}' && command -v xvfb-run");
+    if (probe.ExitCode != 0) Assert.Ignore($"PeaZip's pea is not at {pea} (set PEAZIP_PEA), or xvfb-run is missing.");
+    return pea;
+  }
+
+  private static byte[] LoadVector(string name) {
+    using var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream($"PeaVectors.{name}")
+      ?? throw new InvalidOperationException($"missing embedded vector {name}");
+    using var ms = new MemoryStream();
+    stream.CopyTo(ms);
+    return ms.ToArray();
+  }
+}

Compression.Tests/Pea/ReferenceVectors/README.md

@@ -0,0 +1,25 @@
+# PEA reference archives
+
+Written by PeaZip 11.4.0's `pea` (Linux GTK2 portable build, run headless under `xvfb-run` on
+Ubuntu 24.04 / WSL), never by CompressionWorkbench.
+
+Source tree `/tmp/peasrc`: `hello.txt` (`hello pea\n`), `fox.txt`
+(`The quick brown fox jumps over the lazy dog.\n` × 300), `empty.txt` (0 bytes), `sub/n.txt`
+(`nested\n`), all with mtime 2001-02-03 04:05:06 UTC; `big.bin` is the 3,000,000 bytes
+`(i * 131 + (i >> 7)) mod 256`.
+
+Every archive was made with
+`pea PEA <out> 0 <compression> <volume check> <object check> <stream check> HIDDEN FROMCL /tmp/peasrc/hello.txt /tmp/peasrc/fox.txt /tmp/peasrc/empty.txt /tmp/peasrc/sub`:
+
+| File | Compression | Volume | Object | Stream |
+| --- | --- | --- | --- | --- |
+| `store.pea` | PCOMPRESS0 | CRC32 | CRC32 | NOALGO |
+| `deflate3.pea` | PCOMPRESS1 | ADLER32 | CRC64 | SHA256 |
+| `deflate6.pea` | PCOMPRESS2 | CRC32 | CRC32 | SHA256 |
+| `deflate9.pea` | PCOMPRESS3 | SHA1 | MD5 | SHA512 |
+| `sha3.pea` | PCOMPRESS2 | SHA3_256 | SHA3_512 | BLAKE2B |
+| `misc.pea` | PCOMPRESS2 | RIPEMD160 | WHIRLPOOL | BLAKE2S |
+| `big.pea` | PCOMPRESS2 | CRC32 | CRC32 | SHA256 (input: `big.bin` only; three 1 MiB blocks) |
+
+`pea UNPEA <archive> <dir>/ RESETDATE RESETATTR EXTRACT2DIR HIDDEN "" ""` extracts them under
+`<dir>/<archive name>/`, relative to the objects' common directory.

Hawkynt.Algorithms.Hashing/Algorithms/Whirlpool.cs

@@ -43,7 +43,9 @@ public static byte[] Compute(ReadOnlySpan<byte> data) {
     }
 
     var remaining = data.Length - offset;
-    var finalBlocks = remaining + 1 > 56 ? 128 : 64;
+    // Whirlpool appends 0x80, pads to 32 mod 64 and ends with a 256-bit length: a second block is
+    // needed once the tail and the 0x80 leave fewer than 32 bytes.
+    var finalBlocks = remaining + 1 > 32 ? 128 : 64;
     var final = new byte[finalBlocks];
     data[offset..].CopyTo(final);
     final[remaining] = 0x80;