+ SeqBox (.sbx) container: read and create

Compression.Tests/Compression.Tests.csproj

@@ -130,6 +130,10 @@
     <EmbeddedResource Include="Zfs\ReferenceVectors\*.gz;Zfs\ReferenceVectors\*.manifest">
       <LogicalName>ZfsVectors.%(Filename)%(Extension)</LogicalName>
     </EmbeddedResource>
+    <!-- SeqBox containers written by sbxenc; see SeqBox/ReferenceVectors/README.md. -->
+    <EmbeddedResource Include="SeqBox\ReferenceVectors\*.sbx">
+      <LogicalName>SeqBoxVectors.%(Filename)%(Extension)</LogicalName>
+    </EmbeddedResource>
     <!-- Zoo archives written by Zoo 2.10; see Zoo/ReferenceVectors/README.md. -->
     <EmbeddedResource Include="Zoo\ReferenceVectors\*.zoo">
       <LogicalName>ZooVectors.%(Filename)%(Extension)</LogicalName>

Compression.Tests/ConversionMatrix/ConversionMatrixTests.cs

@@ -126,6 +126,7 @@ public static IEnumerable<TestCaseData> Grid() {
       { "StuffItX", "[single-payload/whole-image target] writer emits an image that re-lists as 0 files" },
       { "Umx", "[single-payload/whole-image target] Unreal package; writer emits a blob that re-lists as 0 files" },
       { "Wbn", "[single-payload/whole-image target] WebBundle writer collapses tree to a single FULL.wbn" },
+      { "SeqBox", "[single-payload/whole-image target] a SeqBox container holds exactly one file; a tree of several is refused" },
       { "Wrapster", "[single-payload/whole-image target] Wrapster MP3 wrapper; one FULL.mp3 + 0/1 frame, not a tree" },
       // -- bucket: name/charset/size constraint --
       { "Bbc", "[name/charset/size constraint] BBC Micro DFS disk; name-synthesizing FS, payload not found by content" },

Compression.Tests/SeqBox/ReferenceVectors/README.md

@@ -0,0 +1,20 @@
+# SeqBox reference containers
+
+Every file here was written by the reference encoder `sbxenc.py` 1.0.2 from
+[MarcoPon/SeqBox](https://github.com/MarcoPon/SeqBox) at revision
+`dee472acfe89f08b7112ce854ca98f2094fbae3b`, run with Python 3.12 on Ubuntu 24.04 (WSL), never by
+CompressionWorkbench.
+
+Sources: `fox.txt` = `The quick brown fox jumps over the lazy dog. ` × 50 (2250 bytes), `noise.bin` =
+the 1000 bytes `(i * 31 + 7) mod 256`, `empty.bin` = 0 bytes; all with mtime 2001-02-03 04:05:06 UTC.
+
+| File | Command | What it pins |
+| --- | --- | --- |
+| `fox-v1.sbx` | `sbxenc.py -uid 0000c0ffee01 fox.txt fox-v1.sbx` | version 1 (512-byte blocks), full metadata block |
+| `noise-v2.sbx` | `sbxenc.py -uid 0000c0ffee02 -sv 2 noise.bin noise-v2.sbx` | version 2 (128-byte blocks) |
+| `fox-v3.sbx` | `sbxenc.py -uid 0000c0ffee03 -sv 3 fox.txt fox-v3.sbx` | version 3 (4096-byte blocks) |
+| `noise-v4.sbx` | `sbxenc.py -uid 0000c0ffee04 -sv 4 noise.bin noise-v4.sbx` | version 4 (488-byte blocks) |
+| `fox-nometa-v1.sbx` | `sbxenc.py -uid 0000c0ffee05 -nm fox.txt fox-nometa-v1.sbx` | no metadata block: payload keeps the 0x1A padding |
+| `empty-v1.sbx` | `sbxenc.py -uid 0000c0ffee06 empty.bin empty-v1.sbx` | metadata block only |
+
+`sbxdec.py -i fox-v1.sbx` reports the expected name, size (2250) and SHA-256.

Compression.Tests/SeqBox/SeqBoxTests.cs

@@ -0,0 +1,203 @@
+using System.Reflection;
+using Compression.Registry;
+using FileFormat.SeqBox;
+
+namespace Compression.Tests.SeqBox;
+
+/// <summary>
+/// SeqBox (.sbx): containers written by the reference sbxenc (SeqBox/ReferenceVectors), our
+/// writer, and the reference sbxdec as a live oracle for what we write.
+/// </summary>
+[TestFixture]
+public class SeqBoxTests {
+
+  private static readonly byte[] FoxText =
+    System.Text.Encoding.ASCII.GetBytes(string.Concat(Enumerable.Repeat("The quick brown fox jumps over the lazy dog. ", 50)));
+
+  private static readonly byte[] Noise = Enumerable.Range(0, 1000).Select(i => (byte)((i * 31 + 7) % 256)).ToArray();
+
+  private static readonly long FoxTime = new DateTimeOffset(2001, 2, 3, 4, 5, 6, TimeSpan.Zero).ToUnixTimeSeconds();
+
+  private string _tmpDir = null!;
+
+  [SetUp]
+  public void Setup() {
+    this._tmpDir = Path.Combine(Path.GetTempPath(), $"cwb_sbx_{Guid.NewGuid():N}");
+    Directory.CreateDirectory(this._tmpDir);
+  }
+
+  [TearDown]
+  public void Teardown() {
+    try { Directory.Delete(this._tmpDir, true); } catch { /* best effort */ }
+  }
+
+  // ── reference containers (sbxenc 1.0.2) ─────────────────────────────
+
+  [TestCase("fox-v1.sbx", 1, "fox.txt")]
+  [TestCase("fox-v3.sbx", 3, "fox.txt")]
+  [TestCase("noise-v2.sbx", 2, "noise.bin")]
+  [TestCase("noise-v4.sbx", 4, "noise.bin")]
+  [Category("ThemVsUs")]
+  public void GivenSbxencContainer_WhenRead_ThenMetadataAndPayloadMatchTheSource(string vector, int version, string name) {
+    var p = SeqBoxContainer.Read(new MemoryStream(LoadVector(vector)));
+
+    Assert.That(p.Version, Is.EqualTo(version));
+    Assert.That(p.Metadata, Is.Not.Null);
+    Assert.That(p.Metadata!.FileName, Is.EqualTo(name));
+    Assert.That(p.Metadata.SbxName, Is.EqualTo(vector));
+    Assert.That(p.Metadata.FileTime, Is.EqualTo(FoxTime));
+    Assert.That(p.Data, Is.EqualTo(name == "fox.txt" ? FoxText : Noise));
+  }
+
+  [Test, Category("ThemVsUs")]
+  public void GivenSbxencContainerWithoutMetadata_WhenRead_ThenPayloadKeepsTheLastBlocksPadding() {
+    var p = SeqBoxContainer.Read(new MemoryStream(LoadVector("fox-nometa-v1.sbx")));
+
+    Assert.That(p.Metadata, Is.Null);
+    Assert.That(p.DataBlocks, Is.EqualTo(5));
+    Assert.That(p.Data.AsSpan(0, FoxText.Length).ToArray(), Is.EqualTo(FoxText));
+    Assert.That(p.Data.Skip(FoxText.Length), Has.All.EqualTo((byte)0x1A));
+  }
+
+  [Test, Category("Boundary")]
+  public void GivenSbxencContainerOfAnEmptyFile_WhenRead_ThenOnlyTheMetadataBlockAndAnEmptyPayload() {
+    var p = SeqBoxContainer.Read(new MemoryStream(LoadVector("empty-v1.sbx")));
+
+    Assert.That(p.DataBlocks, Is.Zero);
+    Assert.That(p.Metadata!.FileSize, Is.Zero);
+    Assert.That(p.Data, Is.Empty);
+  }
+
+  [Test, Category("ThemVsUs")]
+  public void GivenSbxencContainer_WhenListedAndExtractedThroughTheDescriptor_ThenOneNamedEntry() {
+    var d = new SeqBoxFormatDescriptor();
+    var bytes = LoadVector("fox-v1.sbx");
+
+    var list = d.List(new MemoryStream(bytes), null);
+    Assert.That(list.Single().Name, Is.EqualTo("fox.txt"));
+    Assert.That(list.Single().OriginalSize, Is.EqualTo(FoxText.Length));
+
+    d.Extract(new MemoryStream(bytes), this._tmpDir, null, null);
+    Assert.That(File.ReadAllBytes(Path.Combine(this._tmpDir, "fox.txt")), Is.EqualTo(FoxText));
+  }
+
+  // ── our writer ──────────────────────────────────────────────────────
+
+  [TestCase(1, 512)]
+  [TestCase(2, 128)]
+  [TestCase(3, 4096)]
+  [TestCase(4, 488)]
+  [Category("RoundTrip")]
+  public void GivenData_WhenWrittenAtEachVersion_ThenBlockAlignedAndRoundTrips(int version, int blockSize) {
+    var ms = new MemoryStream();
+    SeqBoxContainer.Write(ms, Noise, "noise.bin", version, uid: [0, 0, 0xC0, 0xFF, 0xEE, 0x09]);
+
+    Assert.That(ms.Length % blockSize, Is.Zero);
+    var p = SeqBoxContainer.Read(new MemoryStream(ms.ToArray()));
+    Assert.That(p.Version, Is.EqualTo(version));
+    Assert.That(p.Uid, Is.EqualTo(new byte[] { 0, 0, 0xC0, 0xFF, 0xEE, 0x09 }));
+    Assert.That(p.Data, Is.EqualTo(Noise));
+  }
+
+  [Test, Category("Boundary")]
+  public void GivenPayloadOfExactlyOneBlock_WhenWritten_ThenNoPaddingBlockFollows() {
+    var data = Enumerable.Range(0, 496).Select(i => (byte)i).ToArray();
+    var ms = new MemoryStream();
+    SeqBoxContainer.Write(ms, data, "x.bin");
+
+    Assert.That(ms.Length, Is.EqualTo(2 * 512));
+    Assert.That(SeqBoxContainer.Read(new MemoryStream(ms.ToArray())).Data, Is.EqualTo(data));
+  }
+
+  [Test, Category("Boundary")]
+  public void GivenEmptyFile_WhenWritten_ThenOnlyTheMetadataBlock() {
+    var ms = new MemoryStream();
+    SeqBoxContainer.Write(ms, [], "empty.bin");
+
+    Assert.That(ms.Length, Is.EqualTo(512));
+    Assert.That(SeqBoxContainer.Read(new MemoryStream(ms.ToArray())).Data, Is.Empty);
+  }
+
+  [Test, Category("Exception")]
+  public void GivenTwoFiles_WhenCreated_ThenRefusesBecauseSeqBoxHoldsOneFile() {
+    var d = new SeqBoxFormatDescriptor();
+    Assert.That(() => d.Create(new MemoryStream(),
+      [ArchiveInputInfo.InMemory("a", [1]), ArchiveInputInfo.InMemory("b", [2])], new FormatCreateOptions()),
+      Throws.InstanceOf<NotSupportedException>());
+  }
+
+  [Test, Category("Exception")]
+  public void GivenFlippedDataByte_WhenRead_ThenTheBlockCrcRejectsIt() {
+    var bytes = LoadVector("fox-v1.sbx");
+    bytes[512 * 2 + 100] ^= 0x01;
+
+    Assert.That(() => SeqBoxContainer.Read(new MemoryStream(bytes)), Throws.InstanceOf<InvalidDataException>().With.Message.Contains("CRC"));
+  }
+
+  [Test, Category("Exception")]
+  public void GivenSwappedBlocks_WhenRead_ThenTheSequenceCheckRejectsIt() {
+    var bytes = LoadVector("fox-v1.sbx");
+    var swapped = bytes.ToArray();
+    bytes.AsSpan(512, 512).CopyTo(swapped.AsSpan(1024));
+    bytes.AsSpan(1024, 512).CopyTo(swapped.AsSpan(512));
+
+    Assert.That(() => SeqBoxContainer.Read(new MemoryStream(swapped)), Throws.InstanceOf<InvalidDataException>().With.Message.Contains("order"));
+  }
+
+  [Test, Category("Exception")]
+  public void GivenMissingLastBlock_WhenRead_ThenTheFileSizeCheckRejectsIt() {
+    var bytes = LoadVector("fox-v1.sbx");
+
+    Assert.That(() => SeqBoxContainer.Read(new MemoryStream(bytes[..^512])), Throws.InstanceOf<InvalidDataException>());
+  }
+
+  [Test, Category("Exception")]
+  public void GivenUnknownVersion_WhenRead_ThenRefused() {
+    var bytes = LoadVector("fox-v1.sbx");
+    bytes[3] = 9;
+
+    Assert.That(() => SeqBoxContainer.Read(new MemoryStream(bytes)), Throws.InstanceOf<InvalidDataException>());
+    Assert.That(SeqBoxContainer.IsSeqBox(bytes), Is.False);
+  }
+
+  // ── oracle: the reference decoder reads what we write ───────────────
+
+  [TestCase(1)]
+  [TestCase(2)]
+  [TestCase(3)]
+  [TestCase(4)]
+  [Category("ExternalInterop")]
+  public void GivenOurContainer_WhenSbxdecDecodesIt_ThenHashMatchesAndBytesAreIdentical(int version) {
+    var sbxdec = RequireSbxdec();
+    var big = Enumerable.Range(0, 20_000).Select(i => (byte)(i * 7 + (i >> 8))).ToArray();
+    var path = Path.Combine(this._tmpDir, "ours.sbx");
+    using (var f = File.Create(path))
+      SeqBoxContainer.Write(f, big, "big.bin", version);
+    var outPath = Path.Combine(this._tmpDir, "decoded.bin");
+
+    var r = FsInteropToolbox.RunWsl(
+      $"python3 {sbxdec} -o '{FsInteropToolbox.WinToWsl(path)}' '{FsInteropToolbox.WinToWsl(outPath)}'");
+
+    Assert.That(r.ExitCode, Is.EqualTo(0), $"sbxdec rejected our container:\n{r.StdOut}\n{r.StdErr}");
+    Assert.That(r.StdOut, Does.Contain("hash match!"));
+    Assert.That(File.ReadAllBytes(outPath), Is.EqualTo(big));
+  }
+
+  private static string RequireSbxdec() {
+    if (OperatingSystem.IsWindows() && !FsInteropToolbox.WslAvailable)
+      Assert.Ignore("WSL is not available.");
+    var dir = Environment.GetEnvironmentVariable("SEQBOX_HOME") ?? "/opt/seqbox";
+    var probe = FsInteropToolbox.RunWsl($"test -f '{dir}/sbxdec.py' && command -v python3");
+    if (probe.ExitCode != 0)
+      Assert.Ignore($"SeqBox reference tools not found at {dir} (set SEQBOX_HOME to a checkout of github.com/MarcoPon/SeqBox).");
+    return $"'{dir}/sbxdec.py'";
+  }
+
+  private static byte[] LoadVector(string name) {
+    using var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream($"SeqBoxVectors.{name}")
+      ?? throw new InvalidOperationException($"missing embedded vector {name}");
+    using var ms = new MemoryStream();
+    stream.CopyTo(ms);
+    return ms.ToArray();
+  }
+}